@aglyn/plugins-ai 1.0.0-beta.271 → 1.0.0-beta.272
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +17 -17
- package/src/lib/activity/ai-activity-actions.d.ts +4 -3
- package/src/lib/activity/ai-activity-actions.js +3 -0
- package/src/lib/activity/ai-activity-actions.js.map +1 -1
- package/src/lib/components/ai-site-build-page.component.js +18 -0
- package/src/lib/components/ai-site-build-page.component.js.map +1 -1
- package/src/lib/components/ai-site-import-job.component.d.ts +41 -0
- package/src/lib/components/ai-site-import-job.component.js +707 -0
- package/src/lib/components/ai-site-import-job.component.js.map +1 -0
- package/src/lib/components/ai-site-import-start.component.d.ts +64 -0
- package/src/lib/components/ai-site-import-start.component.js +335 -0
- package/src/lib/components/ai-site-import-start.component.js.map +1 -0
- package/src/lib/components/ai-site-ownership-proof.component.d.ts +42 -0
- package/src/lib/components/ai-site-ownership-proof.component.js +353 -0
- package/src/lib/components/ai-site-ownership-proof.component.js.map +1 -0
- package/src/lib/components/ai-site-start-card.component.js +165 -91
- package/src/lib/components/ai-site-start-card.component.js.map +1 -1
- package/src/lib/jobs/ai-job-drafts.d.ts +14 -0
- package/src/lib/jobs/ai-job-drafts.js +24 -0
- package/src/lib/jobs/ai-job-drafts.js.map +1 -1
- package/src/lib/jobs/ai-job-import-step.d.ts +136 -0
- package/src/lib/jobs/ai-job-import-step.js +809 -0
- package/src/lib/jobs/ai-job-import-step.js.map +1 -0
- package/src/lib/jobs/ai-job-page-language.d.ts +39 -1
- package/src/lib/jobs/ai-job-page-language.js +62 -4
- package/src/lib/jobs/ai-job-page-language.js.map +1 -1
- package/src/lib/jobs/ai-job-page-step.d.ts +2 -2
- package/src/lib/jobs/ai-job-page-step.js +5 -2
- package/src/lib/jobs/ai-job-page-step.js.map +1 -1
- package/src/lib/jobs/ai-job-text-step.d.ts +7 -0
- package/src/lib/jobs/ai-job-text-step.js.map +1 -1
- package/src/lib/jobs/ai-jobs.d.ts +5 -0
- package/src/lib/jobs/ai-jobs.js +15 -3
- package/src/lib/jobs/ai-jobs.js.map +1 -1
- package/src/lib/jobs/ai-layout-owner-photos.d.ts +16 -1
- package/src/lib/jobs/ai-layout-owner-photos.js +47 -0
- package/src/lib/jobs/ai-layout-owner-photos.js.map +1 -1
- package/src/lib/jobs/ai-theme-brand-inputs.d.ts +61 -3
- package/src/lib/jobs/ai-theme-brand-inputs.js +117 -27
- package/src/lib/jobs/ai-theme-brand-inputs.js.map +1 -1
- package/src/lib/model/ai-credit-estimate.d.ts +20 -0
- package/src/lib/model/ai-credit-estimate.js +13 -0
- package/src/lib/model/ai-credit-estimate.js.map +1 -1
- package/src/lib/model/ai-job-activity.d.ts +2 -2
- package/src/lib/model/ai-job-activity.js +6 -4
- package/src/lib/model/ai-job-activity.js.map +1 -1
- package/src/lib/model/ai-job-notice.js +18 -4
- package/src/lib/model/ai-job-notice.js.map +1 -1
- package/src/lib/model/ai-jobs.types.d.ts +29 -2
- package/src/lib/model/ai-jobs.types.js +2 -1
- package/src/lib/model/ai-jobs.types.js.map +1 -1
- package/src/lib/model/ai-site-import-view.d.ts +130 -0
- package/src/lib/model/ai-site-import-view.js +182 -0
- package/src/lib/model/ai-site-import-view.js.map +1 -0
- package/src/lib/model/ai-site-import.d.ts +302 -0
- package/src/lib/model/ai-site-import.js +526 -0
- package/src/lib/model/ai-site-import.js.map +1 -0
- package/src/lib/model/ai-site-start.d.ts +22 -9
- package/src/lib/model/ai-site-start.js +64 -23
- package/src/lib/model/ai-site-start.js.map +1 -1
- package/src/lib/server/ai-jobs-route.js +11 -0
- package/src/lib/server/ai-jobs-route.js.map +1 -1
- package/src/lib/server/ai-site-import-route.d.ts +89 -0
- package/src/lib/server/ai-site-import-route.js +356 -0
- package/src/lib/server/ai-site-import-route.js.map +1 -0
- package/src/lib/server.js +14 -0
- package/src/lib/server.js.map +1 -1
- package/src/lib/site-import/fixtures/site-import-crawl-recordings.d.ts +45 -0
- package/src/lib/site-import/fixtures/site-import-crawl-recordings.js +313 -0
- package/src/lib/site-import/fixtures/site-import-crawl-recordings.js.map +1 -0
- package/src/lib/site-import/fixtures/site-import-wave-recordings.d.ts +50 -0
- package/src/lib/site-import/fixtures/site-import-wave-recordings.js +217 -0
- package/src/lib/site-import/fixtures/site-import-wave-recordings.js.map +1 -0
- package/src/lib/site-import/site-import-crawl.d.ts +163 -0
- package/src/lib/site-import/site-import-crawl.js +467 -0
- package/src/lib/site-import/site-import-crawl.js.map +1 -0
- package/src/lib/site-import/site-import-flag.d.ts +23 -0
- package/src/lib/site-import/site-import-flag.js +23 -0
- package/src/lib/site-import/site-import-flag.js.map +1 -0
- package/src/lib/site-import/site-import-html.d.ts +94 -0
- package/src/lib/site-import/site-import-html.js +462 -0
- package/src/lib/site-import/site-import-html.js.map +1 -0
- package/src/lib/site-import/site-import-images.d.ts +54 -0
- package/src/lib/site-import/site-import-images.js +204 -0
- package/src/lib/site-import/site-import-images.js.map +1 -0
- package/src/lib/site-import/site-import-page.d.ts +48 -0
- package/src/lib/site-import/site-import-page.js +150 -0
- package/src/lib/site-import/site-import-page.js.map +1 -0
- package/src/lib/site-import/site-import-review.d.ts +40 -0
- package/src/lib/site-import/site-import-review.js +149 -0
- package/src/lib/site-import/site-import-review.js.map +1 -0
- package/src/lib/site-import/site-import-robots.d.ts +52 -0
- package/src/lib/site-import/site-import-robots.js +126 -0
- package/src/lib/site-import/site-import-robots.js.map +1 -0
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @license
|
|
3
|
+
* Copyright 2026 Aglyn LLC
|
|
4
|
+
*
|
|
5
|
+
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
6
|
+
* you may not use this file except in compliance with the License.
|
|
7
|
+
* You may obtain a copy of the License at
|
|
8
|
+
*
|
|
9
|
+
* http://www.apache.org/licenses/LICENSE-2.0
|
|
10
|
+
*
|
|
11
|
+
* Unless required by applicable law or agreed to in writing, software
|
|
12
|
+
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
13
|
+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
14
|
+
* See the License for the specific language governing permissions and
|
|
15
|
+
* limitations under the License.
|
|
16
|
+
*/
|
|
17
|
+
/**
|
|
18
|
+
* The "Bring my site" doors (AGL-3974, parent AGL-3973).
|
|
19
|
+
*
|
|
20
|
+
* GET /api/ai/site-import/ownership?orgId= the org's claims
|
|
21
|
+
* POST /api/ai/site-import/ownership { orgId, action, domain, … }
|
|
22
|
+
* action `claim` start a claim (idempotent)
|
|
23
|
+
* action `check`, method `dns` | `meta` look for the record or tag now
|
|
24
|
+
* action `send-code`, email email a code to an address at the domain
|
|
25
|
+
* action `confirm-code`, code check the code
|
|
26
|
+
* POST /api/ai/site-import/crawl { orgId, domain, maxPages? }
|
|
27
|
+
* the site's inventory, for a domain the org has a CURRENT proof for
|
|
28
|
+
*
|
|
29
|
+
* The rungs, in order, each disclosing nothing a lower one would:
|
|
30
|
+
*
|
|
31
|
+
* 401 no bearer token, or a refused one
|
|
32
|
+
* 403 email unverified (an impersonation session is exempt)
|
|
33
|
+
* 400 the request did not name the org (AGL-1934)
|
|
34
|
+
* 403 not a member of that org
|
|
35
|
+
* 404 `release_ai_site_import` is off; a verified staff claim previews
|
|
36
|
+
* 403 `reason: 'role'`: the caller is not the org's owner or an admin
|
|
37
|
+
* 403 `reason: 'plan'`: the org is on the Free plan; import is for every
|
|
38
|
+
* paid plan
|
|
39
|
+
* 423 lockdown
|
|
40
|
+
*
|
|
41
|
+
* Nothing here spends model credits: the crawl reads and returns, and the
|
|
42
|
+
* import job (another lane) decides what becomes a page.
|
|
43
|
+
*/
|
|
44
|
+
import type { DecodedIdToken } from 'firebase-admin/auth';
|
|
45
|
+
import type { AglynOrganization, AglynOrgMember } from '@aglyn/aglyn/foundation/definitions/organization.types';
|
|
46
|
+
import { type DomainOwnershipDeps } from '@aglyn/tenant-data-admin/server/domain-ownership';
|
|
47
|
+
import { type SiteImportCrawlSeams } from '../site-import/site-import-crawl';
|
|
48
|
+
import { SITE_IMPORT_RELEASE_FLAG } from '../site-import/site-import-flag';
|
|
49
|
+
export { SITE_IMPORT_RELEASE_FLAG };
|
|
50
|
+
/** Pages the crawl door reads in one request; the import job reads in waves. */
|
|
51
|
+
export declare const SITE_IMPORT_DOOR_MAX_PAGES = 25;
|
|
52
|
+
/** Crawls one org may run through the door in a day. */
|
|
53
|
+
export declare const SITE_IMPORT_CRAWLS_PER_ORG_PER_DAY = 20;
|
|
54
|
+
/** DNS and home-page checks one org may run in an hour. */
|
|
55
|
+
export declare const SITE_IMPORT_CHECKS_PER_ORG_PER_HOUR = 60;
|
|
56
|
+
export interface SiteImportGateContext {
|
|
57
|
+
uid: string;
|
|
58
|
+
decoded: DecodedIdToken;
|
|
59
|
+
staff: boolean;
|
|
60
|
+
orgId: string;
|
|
61
|
+
org: Partial<AglynOrganization>;
|
|
62
|
+
member: AglynOrgMember | null;
|
|
63
|
+
firestore: FirebaseFirestore.Firestore;
|
|
64
|
+
}
|
|
65
|
+
/** What the doors reach outside the process, replaceable in a spec. */
|
|
66
|
+
export interface SiteImportRouteSeams {
|
|
67
|
+
ownership?: DomainOwnershipDeps;
|
|
68
|
+
crawl?: SiteImportCrawlSeams;
|
|
69
|
+
}
|
|
70
|
+
export declare function siteImportGate(request: Request, rawOrgId: string): Promise<Response | SiteImportGateContext>;
|
|
71
|
+
/**
|
|
72
|
+
* Whether "Bring my site" is offered to this reader (AGL-3976), in the
|
|
73
|
+
* doors' own order, for Create site's choice card: `null` while the flag is
|
|
74
|
+
* off (the card is not drawn), `role` or `plan` for a refusal the workspace
|
|
75
|
+
* can change (the card is drawn disabled with the reason), `ready` otherwise.
|
|
76
|
+
* Read by the jobs list only when the card asks, so no other read pays for it.
|
|
77
|
+
*/
|
|
78
|
+
export declare function siteImportOfferFor(input: {
|
|
79
|
+
orgId: string;
|
|
80
|
+
staff: boolean;
|
|
81
|
+
org: Partial<AglynOrganization> | null;
|
|
82
|
+
member: AglynOrgMember | null;
|
|
83
|
+
}): Promise<'ready' | 'role' | 'plan' | null>;
|
|
84
|
+
/** `GET /api/ai/site-import/ownership?orgId=`. */
|
|
85
|
+
export declare function listSiteImportOwnership(request: Request, seams?: SiteImportRouteSeams): Promise<Response>;
|
|
86
|
+
/** `POST /api/ai/site-import/ownership`. */
|
|
87
|
+
export declare function postSiteImportOwnership(request: Request, seams?: SiteImportRouteSeams): Promise<Response>;
|
|
88
|
+
/** `POST /api/ai/site-import/crawl`. */
|
|
89
|
+
export declare function postSiteImportCrawl(request: Request, seams?: SiteImportRouteSeams): Promise<Response>;
|
|
@@ -0,0 +1,356 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @license
|
|
3
|
+
* Copyright 2026 Aglyn LLC
|
|
4
|
+
*
|
|
5
|
+
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
6
|
+
* you may not use this file except in compliance with the License.
|
|
7
|
+
* You may obtain a copy of the License at
|
|
8
|
+
*
|
|
9
|
+
* http://www.apache.org/licenses/LICENSE-2.0
|
|
10
|
+
*
|
|
11
|
+
* Unless required by applicable law or agreed to in writing, software
|
|
12
|
+
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
13
|
+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
14
|
+
* See the License for the specific language governing permissions and
|
|
15
|
+
* limitations under the License.
|
|
16
|
+
*/ /**
|
|
17
|
+
* The "Bring my site" doors (AGL-3974, parent AGL-3973).
|
|
18
|
+
*
|
|
19
|
+
* GET /api/ai/site-import/ownership?orgId= the org's claims
|
|
20
|
+
* POST /api/ai/site-import/ownership { orgId, action, domain, … }
|
|
21
|
+
* action `claim` start a claim (idempotent)
|
|
22
|
+
* action `check`, method `dns` | `meta` look for the record or tag now
|
|
23
|
+
* action `send-code`, email email a code to an address at the domain
|
|
24
|
+
* action `confirm-code`, code check the code
|
|
25
|
+
* POST /api/ai/site-import/crawl { orgId, domain, maxPages? }
|
|
26
|
+
* the site's inventory, for a domain the org has a CURRENT proof for
|
|
27
|
+
*
|
|
28
|
+
* The rungs, in order, each disclosing nothing a lower one would:
|
|
29
|
+
*
|
|
30
|
+
* 401 no bearer token, or a refused one
|
|
31
|
+
* 403 email unverified (an impersonation session is exempt)
|
|
32
|
+
* 400 the request did not name the org (AGL-1934)
|
|
33
|
+
* 403 not a member of that org
|
|
34
|
+
* 404 `release_ai_site_import` is off; a verified staff claim previews
|
|
35
|
+
* 403 `reason: 'role'`: the caller is not the org's owner or an admin
|
|
36
|
+
* 403 `reason: 'plan'`: the org is on the Free plan; import is for every
|
|
37
|
+
* paid plan
|
|
38
|
+
* 423 lockdown
|
|
39
|
+
*
|
|
40
|
+
* Nothing here spends model credits: the crawl reads and returns, and the
|
|
41
|
+
* import job (another lane) decides what becomes a page.
|
|
42
|
+
*/ import { _ as _extends } from "@swc/helpers/_/_extends";
|
|
43
|
+
import { resolveEffectivePlan, resolveOrgEntitlements } from "@aglyn/aglyn/app-utils/plan-entitlements";
|
|
44
|
+
import { addAdminAudit } from "@aglyn/tenant-data-admin/server/admin-audit-write";
|
|
45
|
+
import { checkDomainOwnership, confirmDomainOwnershipCode, DOMAIN_OWNERSHIP_META_NAME, isDomainOwnershipProven, issueDomainOwnershipClaim, listDomainOwnershipClaims, normalizeOwnershipDomain, sendDomainOwnershipCode } from "@aglyn/tenant-data-admin/server/domain-ownership";
|
|
46
|
+
import { emailUnverifiedResponse, firebaseAdmin, isImpersonationSession } from "@aglyn/tenant-data-admin/server/firebase-admin";
|
|
47
|
+
import { invalidIdTokenResponse } from "@aglyn/tenant-data-admin/server/id-token-refusal";
|
|
48
|
+
import { lockdownRefusal } from "@aglyn/tenant-data-admin/server/lockdown";
|
|
49
|
+
import { getOrgForUser } from "@aglyn/tenant-data-admin/server/organizations";
|
|
50
|
+
import { consumeRateLimit } from "@aglyn/tenant-data-admin/server/rate-limit-store";
|
|
51
|
+
import { isServerReleaseFlagOnForOrg } from "@aglyn/tenant-data-admin/server/release-flags";
|
|
52
|
+
import { FieldValue } from "firebase-admin/firestore";
|
|
53
|
+
import { crawlSiteForImport, readSiteVerificationMeta } from "../site-import/site-import-crawl.js";
|
|
54
|
+
import { SITE_IMPORT_RELEASE_FLAG } from "../site-import/site-import-flag.js";
|
|
55
|
+
export { SITE_IMPORT_RELEASE_FLAG };
|
|
56
|
+
/** Pages the crawl door reads in one request; the import job reads in waves. */ export const SITE_IMPORT_DOOR_MAX_PAGES = 25;
|
|
57
|
+
/** Crawls one org may run through the door in a day. */ export const SITE_IMPORT_CRAWLS_PER_ORG_PER_DAY = 20;
|
|
58
|
+
/** DNS and home-page checks one org may run in an hour. */ export const SITE_IMPORT_CHECKS_PER_ORG_PER_HOUR = 60;
|
|
59
|
+
/** The door's own clock for a crawl, inside the route's `maxDuration`. */ const SITE_IMPORT_DOOR_TIME_MS = 45000;
|
|
60
|
+
function bearerToken(request) {
|
|
61
|
+
var _request_headers_get;
|
|
62
|
+
const authorization = (_request_headers_get = request.headers.get('authorization')) != null ? _request_headers_get : '';
|
|
63
|
+
return authorization.startsWith('Bearer ') ? authorization.slice('Bearer '.length) : undefined;
|
|
64
|
+
}
|
|
65
|
+
export async function siteImportGate(request, rawOrgId) {
|
|
66
|
+
var _resolved_member, _resolved_org;
|
|
67
|
+
const idToken = bearerToken(request);
|
|
68
|
+
if (!idToken) return Response.json({
|
|
69
|
+
error: 'Unauthenticated'
|
|
70
|
+
}, {
|
|
71
|
+
status: 401
|
|
72
|
+
});
|
|
73
|
+
const app = firebaseAdmin.app();
|
|
74
|
+
let decoded;
|
|
75
|
+
try {
|
|
76
|
+
decoded = await app.auth().verifyIdToken(idToken);
|
|
77
|
+
} catch (error) {
|
|
78
|
+
const unauthenticated = invalidIdTokenResponse(error);
|
|
79
|
+
if (unauthenticated) return unauthenticated;
|
|
80
|
+
throw error;
|
|
81
|
+
}
|
|
82
|
+
if (!decoded.email_verified && !isImpersonationSession(decoded)) return emailUnverifiedResponse();
|
|
83
|
+
const staff = decoded['staff'] === true;
|
|
84
|
+
const orgId = String(rawOrgId != null ? rawOrgId : '').trim();
|
|
85
|
+
if (!orgId) {
|
|
86
|
+
return Response.json({
|
|
87
|
+
error: 'Open a workspace before using this feature'
|
|
88
|
+
}, {
|
|
89
|
+
status: 400
|
|
90
|
+
});
|
|
91
|
+
}
|
|
92
|
+
const resolved = await getOrgForUser(decoded.uid, orgId);
|
|
93
|
+
if (!resolved || resolved.orgId !== orgId) {
|
|
94
|
+
return Response.json({
|
|
95
|
+
error: 'You are not a member of that organization'
|
|
96
|
+
}, {
|
|
97
|
+
status: 403
|
|
98
|
+
});
|
|
99
|
+
}
|
|
100
|
+
if (!staff && !await isServerReleaseFlagOnForOrg(SITE_IMPORT_RELEASE_FLAG, orgId)) {
|
|
101
|
+
return Response.json({
|
|
102
|
+
error: 'Not found'
|
|
103
|
+
}, {
|
|
104
|
+
status: 404
|
|
105
|
+
});
|
|
106
|
+
}
|
|
107
|
+
const member = (_resolved_member = resolved.member) != null ? _resolved_member : null;
|
|
108
|
+
if (!member || member.role !== 'owner' && member.role !== 'admin') {
|
|
109
|
+
return Response.json({
|
|
110
|
+
error: 'Only the workspace owner or an admin can bring a site in.',
|
|
111
|
+
reason: 'role'
|
|
112
|
+
}, {
|
|
113
|
+
status: 403
|
|
114
|
+
});
|
|
115
|
+
}
|
|
116
|
+
const org = (_resolved_org = resolved.org) != null ? _resolved_org : {};
|
|
117
|
+
if (resolveEffectivePlan(org) === 'free') {
|
|
118
|
+
return Response.json({
|
|
119
|
+
error: 'Bringing an existing site in is included on every paid plan.',
|
|
120
|
+
reason: 'plan'
|
|
121
|
+
}, {
|
|
122
|
+
status: 403
|
|
123
|
+
});
|
|
124
|
+
}
|
|
125
|
+
const locked = await lockdownRefusal({
|
|
126
|
+
request,
|
|
127
|
+
staff,
|
|
128
|
+
uid: decoded.uid,
|
|
129
|
+
org: org
|
|
130
|
+
});
|
|
131
|
+
if (locked) return locked;
|
|
132
|
+
return {
|
|
133
|
+
uid: decoded.uid,
|
|
134
|
+
decoded,
|
|
135
|
+
staff,
|
|
136
|
+
orgId,
|
|
137
|
+
org,
|
|
138
|
+
member,
|
|
139
|
+
firestore: app.firestore()
|
|
140
|
+
};
|
|
141
|
+
}
|
|
142
|
+
/**
|
|
143
|
+
* Whether "Bring my site" is offered to this reader (AGL-3976), in the
|
|
144
|
+
* doors' own order, for Create site's choice card: `null` while the flag is
|
|
145
|
+
* off (the card is not drawn), `role` or `plan` for a refusal the workspace
|
|
146
|
+
* can change (the card is drawn disabled with the reason), `ready` otherwise.
|
|
147
|
+
* Read by the jobs list only when the card asks, so no other read pays for it.
|
|
148
|
+
*/ export async function siteImportOfferFor(input) {
|
|
149
|
+
var _input_org;
|
|
150
|
+
try {
|
|
151
|
+
if (!input.staff && !await isServerReleaseFlagOnForOrg(SITE_IMPORT_RELEASE_FLAG, input.orgId)) return null;
|
|
152
|
+
} catch (unused) {
|
|
153
|
+
return null;
|
|
154
|
+
}
|
|
155
|
+
if (!input.member || input.member.role !== 'owner' && input.member.role !== 'admin') return 'role';
|
|
156
|
+
if (resolveEffectivePlan((_input_org = input.org) != null ? _input_org : {}) === 'free') return 'plan';
|
|
157
|
+
return 'ready';
|
|
158
|
+
}
|
|
159
|
+
async function audit(gate, action, domain, after) {
|
|
160
|
+
try {
|
|
161
|
+
var _gate_decoded_email;
|
|
162
|
+
await addAdminAudit(gate.firestore, {
|
|
163
|
+
actorUid: gate.uid,
|
|
164
|
+
actorEmail: (_gate_decoded_email = gate.decoded.email) != null ? _gate_decoded_email : null,
|
|
165
|
+
action,
|
|
166
|
+
target: `orgs/${gate.orgId}/domainOwnershipClaims/${domain}`,
|
|
167
|
+
after: _extends({}, after, gate.staff ? {
|
|
168
|
+
staff: true
|
|
169
|
+
} : {}),
|
|
170
|
+
at: FieldValue.serverTimestamp()
|
|
171
|
+
});
|
|
172
|
+
} catch (error) {
|
|
173
|
+
console.error('site import audit write failed', {
|
|
174
|
+
orgId: gate.orgId,
|
|
175
|
+
action,
|
|
176
|
+
error
|
|
177
|
+
});
|
|
178
|
+
}
|
|
179
|
+
}
|
|
180
|
+
function ownershipDeps(seams) {
|
|
181
|
+
return _extends({
|
|
182
|
+
readVerificationMeta: (domain)=>readSiteVerificationMeta(domain, DOMAIN_OWNERSHIP_META_NAME, seams.crawl)
|
|
183
|
+
}, seams.ownership);
|
|
184
|
+
}
|
|
185
|
+
const tooMany = (error)=>Response.json({
|
|
186
|
+
error,
|
|
187
|
+
reason: 'rate'
|
|
188
|
+
}, {
|
|
189
|
+
status: 429
|
|
190
|
+
});
|
|
191
|
+
/** `GET /api/ai/site-import/ownership?orgId=`. */ export async function listSiteImportOwnership(request, seams = {}) {
|
|
192
|
+
var _searchParams_get;
|
|
193
|
+
const orgId = (_searchParams_get = new URL(request.url).searchParams.get('orgId')) != null ? _searchParams_get : '';
|
|
194
|
+
const gate = await siteImportGate(request, orgId);
|
|
195
|
+
if (gate instanceof Response) return gate;
|
|
196
|
+
const claims = await listDomainOwnershipClaims(gate.orgId, ownershipDeps(seams));
|
|
197
|
+
return Response.json({
|
|
198
|
+
claims
|
|
199
|
+
});
|
|
200
|
+
}
|
|
201
|
+
/** `POST /api/ai/site-import/ownership`. */ export async function postSiteImportOwnership(request, seams = {}) {
|
|
202
|
+
var _body_orgId, _body_action, _body_domain;
|
|
203
|
+
const body = await request.json().catch(()=>null);
|
|
204
|
+
if (!body || typeof body !== 'object') {
|
|
205
|
+
return Response.json({
|
|
206
|
+
error: 'Send a JSON body'
|
|
207
|
+
}, {
|
|
208
|
+
status: 400
|
|
209
|
+
});
|
|
210
|
+
}
|
|
211
|
+
const gate = await siteImportGate(request, String((_body_orgId = body['orgId']) != null ? _body_orgId : ''));
|
|
212
|
+
if (gate instanceof Response) return gate;
|
|
213
|
+
const deps = ownershipDeps(seams);
|
|
214
|
+
const action = String((_body_action = body['action']) != null ? _body_action : '');
|
|
215
|
+
const domain = String((_body_domain = body['domain']) != null ? _body_domain : '');
|
|
216
|
+
if (action === 'claim') {
|
|
217
|
+
const { claim, error } = await issueDomainOwnershipClaim(gate.orgId, domain, gate.uid, deps);
|
|
218
|
+
if (!claim) return Response.json({
|
|
219
|
+
error
|
|
220
|
+
}, {
|
|
221
|
+
status: 400
|
|
222
|
+
});
|
|
223
|
+
await audit(gate, 'site-import.ownership.claim', claim.domain, {
|
|
224
|
+
status: claim.status
|
|
225
|
+
});
|
|
226
|
+
return Response.json({
|
|
227
|
+
claim
|
|
228
|
+
});
|
|
229
|
+
}
|
|
230
|
+
if (action === 'check') {
|
|
231
|
+
const method = body['method'];
|
|
232
|
+
if (method !== 'dns' && method !== 'meta') {
|
|
233
|
+
return Response.json({
|
|
234
|
+
error: 'method must be dns or meta'
|
|
235
|
+
}, {
|
|
236
|
+
status: 400
|
|
237
|
+
});
|
|
238
|
+
}
|
|
239
|
+
const window = await consumeRateLimit(`site-import-ownership-check:${gate.orgId}`, {
|
|
240
|
+
limit: SITE_IMPORT_CHECKS_PER_ORG_PER_HOUR,
|
|
241
|
+
windowMs: 60 * 60 * 1000
|
|
242
|
+
});
|
|
243
|
+
if (!window.allowed) return tooMany('Too many checks. Try again in an hour.');
|
|
244
|
+
const { result, error } = await checkDomainOwnership(gate.orgId, domain, method, deps);
|
|
245
|
+
if (!result) return Response.json({
|
|
246
|
+
error
|
|
247
|
+
}, {
|
|
248
|
+
status: 400
|
|
249
|
+
});
|
|
250
|
+
if (result.status === 'proven') {
|
|
251
|
+
await audit(gate, 'site-import.ownership.proven', result.claim.domain, {
|
|
252
|
+
method,
|
|
253
|
+
detail: result.claim.provenDetail
|
|
254
|
+
});
|
|
255
|
+
}
|
|
256
|
+
return Response.json({
|
|
257
|
+
result
|
|
258
|
+
});
|
|
259
|
+
}
|
|
260
|
+
if (action === 'send-code') {
|
|
261
|
+
var _body_email;
|
|
262
|
+
const sent = await sendDomainOwnershipCode(gate.orgId, domain, String((_body_email = body['email']) != null ? _body_email : ''), {
|
|
263
|
+
org: gate.org
|
|
264
|
+
}, deps);
|
|
265
|
+
if (!sent.claim) return Response.json({
|
|
266
|
+
error: sent.error
|
|
267
|
+
}, {
|
|
268
|
+
status: sent.status
|
|
269
|
+
});
|
|
270
|
+
await audit(gate, 'site-import.ownership.code-sent', sent.claim.domain, {
|
|
271
|
+
address: sent.claim.email.pendingAddress
|
|
272
|
+
});
|
|
273
|
+
return Response.json({
|
|
274
|
+
claim: sent.claim
|
|
275
|
+
});
|
|
276
|
+
}
|
|
277
|
+
if (action === 'confirm-code') {
|
|
278
|
+
var _body_code;
|
|
279
|
+
const { result, error } = await confirmDomainOwnershipCode(gate.orgId, domain, String((_body_code = body['code']) != null ? _body_code : ''), deps);
|
|
280
|
+
if (!result) return Response.json({
|
|
281
|
+
error
|
|
282
|
+
}, {
|
|
283
|
+
status: 400
|
|
284
|
+
});
|
|
285
|
+
if (result.status === 'proven') {
|
|
286
|
+
await audit(gate, 'site-import.ownership.proven', result.claim.domain, {
|
|
287
|
+
method: 'email',
|
|
288
|
+
detail: result.claim.provenDetail
|
|
289
|
+
});
|
|
290
|
+
}
|
|
291
|
+
return Response.json({
|
|
292
|
+
result
|
|
293
|
+
});
|
|
294
|
+
}
|
|
295
|
+
return Response.json({
|
|
296
|
+
error: 'action must be claim, check, send-code or confirm-code'
|
|
297
|
+
}, {
|
|
298
|
+
status: 400
|
|
299
|
+
});
|
|
300
|
+
}
|
|
301
|
+
/** `POST /api/ai/site-import/crawl`. */ export async function postSiteImportCrawl(request, seams = {}) {
|
|
302
|
+
var _body_orgId, _body_domain;
|
|
303
|
+
const body = await request.json().catch(()=>null);
|
|
304
|
+
if (!body || typeof body !== 'object') {
|
|
305
|
+
return Response.json({
|
|
306
|
+
error: 'Send a JSON body'
|
|
307
|
+
}, {
|
|
308
|
+
status: 400
|
|
309
|
+
});
|
|
310
|
+
}
|
|
311
|
+
const gate = await siteImportGate(request, String((_body_orgId = body['orgId']) != null ? _body_orgId : ''));
|
|
312
|
+
if (gate instanceof Response) return gate;
|
|
313
|
+
const { domain, error } = normalizeOwnershipDomain(String((_body_domain = body['domain']) != null ? _body_domain : ''));
|
|
314
|
+
if (!domain) return Response.json({
|
|
315
|
+
error
|
|
316
|
+
}, {
|
|
317
|
+
status: 400
|
|
318
|
+
});
|
|
319
|
+
// Ownership BEFORE any page is read: a crawl of a site the org has not
|
|
320
|
+
// proved is exactly the clone this whole feature is built to refuse.
|
|
321
|
+
if (!await isDomainOwnershipProven(gate.orgId, domain, ownershipDeps(seams))) {
|
|
322
|
+
return Response.json({
|
|
323
|
+
error: 'Verify that you own this site first.',
|
|
324
|
+
reason: 'ownership'
|
|
325
|
+
}, {
|
|
326
|
+
status: 403
|
|
327
|
+
});
|
|
328
|
+
}
|
|
329
|
+
const window = await consumeRateLimit(`site-import-crawl:${gate.orgId}`, {
|
|
330
|
+
limit: SITE_IMPORT_CRAWLS_PER_ORG_PER_DAY,
|
|
331
|
+
windowMs: 24 * 60 * 60 * 1000
|
|
332
|
+
});
|
|
333
|
+
if (!window.allowed) return tooMany('This workspace has read sites enough times today. Try again tomorrow.');
|
|
334
|
+
const planPages = resolveOrgEntitlements(gate.org).screensPerHost;
|
|
335
|
+
const asked = Number(body['maxPages']);
|
|
336
|
+
const maxPages = Math.max(1, Math.min(SITE_IMPORT_DOOR_MAX_PAGES, Number.isFinite(planPages) && planPages > 0 ? planPages : SITE_IMPORT_DOOR_MAX_PAGES, Number.isFinite(asked) && asked > 0 ? Math.floor(asked) : SITE_IMPORT_DOOR_MAX_PAGES));
|
|
337
|
+
const inventory = await crawlSiteForImport({
|
|
338
|
+
domain,
|
|
339
|
+
maxPages,
|
|
340
|
+
timeBudgetMs: SITE_IMPORT_DOOR_TIME_MS,
|
|
341
|
+
signal: request.signal
|
|
342
|
+
}, seams.crawl);
|
|
343
|
+
await audit(gate, 'site-import.crawl', domain, {
|
|
344
|
+
pages: inventory.pages.length,
|
|
345
|
+
skipped: inventory.skipped.length + inventory.skippedOverflow,
|
|
346
|
+
loginPagesRefused: inventory.skipped.filter((entry)=>entry.reason === 'login-form').length,
|
|
347
|
+
stoppedBy: inventory.stoppedBy,
|
|
348
|
+
refused: inventory.refused,
|
|
349
|
+
bytesRead: inventory.bytesRead
|
|
350
|
+
});
|
|
351
|
+
return Response.json({
|
|
352
|
+
inventory
|
|
353
|
+
});
|
|
354
|
+
}
|
|
355
|
+
|
|
356
|
+
//# sourceMappingURL=ai-site-import-route.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["../../../../../../../libs/plugins/ai/src/lib/server/ai-site-import-route.ts"],"sourcesContent":["/**\n * @license\n * Copyright 2026 Aglyn LLC\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n/**\n * The \"Bring my site\" doors (AGL-3974, parent AGL-3973).\n *\n * GET /api/ai/site-import/ownership?orgId= the org's claims\n * POST /api/ai/site-import/ownership { orgId, action, domain, … }\n * action `claim` start a claim (idempotent)\n * action `check`, method `dns` | `meta` look for the record or tag now\n * action `send-code`, email email a code to an address at the domain\n * action `confirm-code`, code check the code\n * POST /api/ai/site-import/crawl { orgId, domain, maxPages? }\n * the site's inventory, for a domain the org has a CURRENT proof for\n *\n * The rungs, in order, each disclosing nothing a lower one would:\n *\n * 401 no bearer token, or a refused one\n * 403 email unverified (an impersonation session is exempt)\n * 400 the request did not name the org (AGL-1934)\n * 403 not a member of that org\n * 404 `release_ai_site_import` is off; a verified staff claim previews\n * 403 `reason: 'role'`: the caller is not the org's owner or an admin\n * 403 `reason: 'plan'`: the org is on the Free plan; import is for every\n * paid plan\n * 423 lockdown\n *\n * Nothing here spends model credits: the crawl reads and returns, and the\n * import job (another lane) decides what becomes a page.\n */\n\nimport type { DecodedIdToken } from 'firebase-admin/auth'\nimport { resolveEffectivePlan, resolveOrgEntitlements } from '@aglyn/aglyn/app-utils/plan-entitlements'\nimport type { AglynOrganization, AglynOrgMember } from '@aglyn/aglyn/foundation/definitions/organization.types'\nimport { addAdminAudit } from '@aglyn/tenant-data-admin/server/admin-audit-write'\nimport {\n checkDomainOwnership,\n confirmDomainOwnershipCode,\n DOMAIN_OWNERSHIP_META_NAME,\n isDomainOwnershipProven,\n issueDomainOwnershipClaim,\n listDomainOwnershipClaims,\n normalizeOwnershipDomain,\n sendDomainOwnershipCode,\n type DomainOwnershipDeps,\n} from '@aglyn/tenant-data-admin/server/domain-ownership'\nimport {\n emailUnverifiedResponse,\n firebaseAdmin,\n isImpersonationSession,\n} from '@aglyn/tenant-data-admin/server/firebase-admin'\nimport { invalidIdTokenResponse } from '@aglyn/tenant-data-admin/server/id-token-refusal'\nimport { lockdownRefusal } from '@aglyn/tenant-data-admin/server/lockdown'\nimport { getOrgForUser } from '@aglyn/tenant-data-admin/server/organizations'\nimport { consumeRateLimit } from '@aglyn/tenant-data-admin/server/rate-limit-store'\nimport { isServerReleaseFlagOnForOrg } from '@aglyn/tenant-data-admin/server/release-flags'\nimport { FieldValue } from 'firebase-admin/firestore'\nimport {\n crawlSiteForImport,\n readSiteVerificationMeta,\n type SiteImportCrawlSeams,\n} from '../site-import/site-import-crawl'\nimport { SITE_IMPORT_RELEASE_FLAG } from '../site-import/site-import-flag'\n\nexport { SITE_IMPORT_RELEASE_FLAG }\n\n/** Pages the crawl door reads in one request; the import job reads in waves. */\nexport const SITE_IMPORT_DOOR_MAX_PAGES = 25\n\n/** Crawls one org may run through the door in a day. */\nexport const SITE_IMPORT_CRAWLS_PER_ORG_PER_DAY = 20\n\n/** DNS and home-page checks one org may run in an hour. */\nexport const SITE_IMPORT_CHECKS_PER_ORG_PER_HOUR = 60\n\n/** The door's own clock for a crawl, inside the route's `maxDuration`. */\nconst SITE_IMPORT_DOOR_TIME_MS = 45_000\n\nexport interface SiteImportGateContext {\n uid: string\n decoded: DecodedIdToken\n staff: boolean\n orgId: string\n org: Partial<AglynOrganization>\n member: AglynOrgMember | null\n firestore: FirebaseFirestore.Firestore\n}\n\n/** What the doors reach outside the process, replaceable in a spec. */\nexport interface SiteImportRouteSeams {\n ownership?: DomainOwnershipDeps\n crawl?: SiteImportCrawlSeams\n}\n\nfunction bearerToken(request: Request): string | undefined {\n const authorization = request.headers.get('authorization') ?? ''\n return authorization.startsWith('Bearer ') ? authorization.slice('Bearer '.length) : undefined\n}\n\nexport async function siteImportGate(\n request: Request,\n rawOrgId: string,\n): Promise<Response | SiteImportGateContext> {\n const idToken = bearerToken(request)\n if (!idToken) return Response.json({ error: 'Unauthenticated' }, { status: 401 })\n const app = firebaseAdmin.app()\n let decoded: DecodedIdToken\n try {\n decoded = await app.auth().verifyIdToken(idToken)\n } catch (error) {\n const unauthenticated = invalidIdTokenResponse(error)\n if (unauthenticated) return unauthenticated\n throw error\n }\n if (!decoded.email_verified && !isImpersonationSession(decoded)) return emailUnverifiedResponse()\n const staff = decoded['staff'] === true\n const orgId = String(rawOrgId ?? '').trim()\n if (!orgId) {\n return Response.json({ error: 'Open a workspace before using this feature' }, { status: 400 })\n }\n const resolved = await getOrgForUser(decoded.uid, orgId)\n if (!resolved || resolved.orgId !== orgId) {\n return Response.json({ error: 'You are not a member of that organization' }, { status: 403 })\n }\n if (!staff && !(await isServerReleaseFlagOnForOrg(SITE_IMPORT_RELEASE_FLAG, orgId))) {\n return Response.json({ error: 'Not found' }, { status: 404 })\n }\n const member = resolved.member ?? null\n if (!member || (member.role !== 'owner' && member.role !== 'admin')) {\n return Response.json(\n { error: 'Only the workspace owner or an admin can bring a site in.', reason: 'role' },\n { status: 403 },\n )\n }\n const org = resolved.org ?? {}\n if (resolveEffectivePlan(org) === 'free') {\n return Response.json(\n { error: 'Bringing an existing site in is included on every paid plan.', reason: 'plan' },\n { status: 403 },\n )\n }\n const locked = await lockdownRefusal({\n request,\n staff,\n uid: decoded.uid,\n org: org as Record<string, unknown>,\n })\n if (locked) return locked\n return { uid: decoded.uid, decoded, staff, orgId, org, member, firestore: app.firestore() }\n}\n\n/**\n * Whether \"Bring my site\" is offered to this reader (AGL-3976), in the\n * doors' own order, for Create site's choice card: `null` while the flag is\n * off (the card is not drawn), `role` or `plan` for a refusal the workspace\n * can change (the card is drawn disabled with the reason), `ready` otherwise.\n * Read by the jobs list only when the card asks, so no other read pays for it.\n */\nexport async function siteImportOfferFor(input: {\n orgId: string\n staff: boolean\n org: Partial<AglynOrganization> | null\n member: AglynOrgMember | null\n}): Promise<'ready' | 'role' | 'plan' | null> {\n try {\n if (!input.staff && !(await isServerReleaseFlagOnForOrg(SITE_IMPORT_RELEASE_FLAG, input.orgId))) return null\n } catch {\n return null\n }\n if (!input.member || (input.member.role !== 'owner' && input.member.role !== 'admin')) return 'role'\n if (resolveEffectivePlan(input.org ?? {}) === 'free') return 'plan'\n return 'ready'\n}\n\nasync function audit(\n gate: SiteImportGateContext,\n action: string,\n domain: string,\n after: Record<string, unknown>,\n): Promise<void> {\n try {\n await addAdminAudit(gate.firestore, {\n actorUid: gate.uid,\n actorEmail: gate.decoded.email ?? null,\n action,\n target: `orgs/${gate.orgId}/domainOwnershipClaims/${domain}`,\n after: { ...after, ...(gate.staff ? { staff: true } : {}) },\n at: FieldValue.serverTimestamp(),\n })\n } catch (error) {\n console.error('site import audit write failed', { orgId: gate.orgId, action, error })\n }\n}\n\nfunction ownershipDeps(seams: SiteImportRouteSeams): DomainOwnershipDeps {\n return {\n readVerificationMeta: (domain) =>\n readSiteVerificationMeta(domain, DOMAIN_OWNERSHIP_META_NAME, seams.crawl),\n ...seams.ownership,\n }\n}\n\nconst tooMany = (error: string) => Response.json({ error, reason: 'rate' }, { status: 429 })\n\n/** `GET /api/ai/site-import/ownership?orgId=`. */\nexport async function listSiteImportOwnership(\n request: Request,\n seams: SiteImportRouteSeams = {},\n): Promise<Response> {\n const orgId = new URL(request.url).searchParams.get('orgId') ?? ''\n const gate = await siteImportGate(request, orgId)\n if (gate instanceof Response) return gate\n const claims = await listDomainOwnershipClaims(gate.orgId, ownershipDeps(seams))\n return Response.json({ claims })\n}\n\n/** `POST /api/ai/site-import/ownership`. */\nexport async function postSiteImportOwnership(\n request: Request,\n seams: SiteImportRouteSeams = {},\n): Promise<Response> {\n const body = (await request.json().catch(() => null)) as Record<string, unknown> | null\n if (!body || typeof body !== 'object') {\n return Response.json({ error: 'Send a JSON body' }, { status: 400 })\n }\n const gate = await siteImportGate(request, String(body['orgId'] ?? ''))\n if (gate instanceof Response) return gate\n const deps = ownershipDeps(seams)\n const action = String(body['action'] ?? '')\n const domain = String(body['domain'] ?? '')\n\n if (action === 'claim') {\n const { claim, error } = await issueDomainOwnershipClaim(gate.orgId, domain, gate.uid, deps)\n if (!claim) return Response.json({ error }, { status: 400 })\n await audit(gate, 'site-import.ownership.claim', claim.domain, { status: claim.status })\n return Response.json({ claim })\n }\n\n if (action === 'check') {\n const method = body['method']\n if (method !== 'dns' && method !== 'meta') {\n return Response.json({ error: 'method must be dns or meta' }, { status: 400 })\n }\n const window = await consumeRateLimit(`site-import-ownership-check:${gate.orgId}`, {\n limit: SITE_IMPORT_CHECKS_PER_ORG_PER_HOUR,\n windowMs: 60 * 60 * 1000,\n })\n if (!window.allowed) return tooMany('Too many checks. Try again in an hour.')\n const { result, error } = await checkDomainOwnership(gate.orgId, domain, method, deps)\n if (!result) return Response.json({ error }, { status: 400 })\n if (result.status === 'proven') {\n await audit(gate, 'site-import.ownership.proven', result.claim.domain, {\n method,\n detail: result.claim.provenDetail,\n })\n }\n return Response.json({ result })\n }\n\n if (action === 'send-code') {\n const sent = await sendDomainOwnershipCode(\n gate.orgId,\n domain,\n String(body['email'] ?? ''),\n { org: gate.org as Record<string, unknown> },\n deps,\n )\n if (!sent.claim) return Response.json({ error: sent.error }, { status: sent.status })\n await audit(gate, 'site-import.ownership.code-sent', sent.claim.domain, {\n address: sent.claim.email.pendingAddress,\n })\n return Response.json({ claim: sent.claim })\n }\n\n if (action === 'confirm-code') {\n const { result, error } = await confirmDomainOwnershipCode(gate.orgId, domain, String(body['code'] ?? ''), deps)\n if (!result) return Response.json({ error }, { status: 400 })\n if (result.status === 'proven') {\n await audit(gate, 'site-import.ownership.proven', result.claim.domain, {\n method: 'email',\n detail: result.claim.provenDetail,\n })\n }\n return Response.json({ result })\n }\n\n return Response.json(\n { error: 'action must be claim, check, send-code or confirm-code' },\n { status: 400 },\n )\n}\n\n/** `POST /api/ai/site-import/crawl`. */\nexport async function postSiteImportCrawl(\n request: Request,\n seams: SiteImportRouteSeams = {},\n): Promise<Response> {\n const body = (await request.json().catch(() => null)) as Record<string, unknown> | null\n if (!body || typeof body !== 'object') {\n return Response.json({ error: 'Send a JSON body' }, { status: 400 })\n }\n const gate = await siteImportGate(request, String(body['orgId'] ?? ''))\n if (gate instanceof Response) return gate\n const { domain, error } = normalizeOwnershipDomain(String(body['domain'] ?? ''))\n if (!domain) return Response.json({ error }, { status: 400 })\n // Ownership BEFORE any page is read: a crawl of a site the org has not\n // proved is exactly the clone this whole feature is built to refuse.\n if (!(await isDomainOwnershipProven(gate.orgId, domain, ownershipDeps(seams)))) {\n return Response.json(\n { error: 'Verify that you own this site first.', reason: 'ownership' },\n { status: 403 },\n )\n }\n const window = await consumeRateLimit(`site-import-crawl:${gate.orgId}`, {\n limit: SITE_IMPORT_CRAWLS_PER_ORG_PER_DAY,\n windowMs: 24 * 60 * 60 * 1000,\n })\n if (!window.allowed) return tooMany('This workspace has read sites enough times today. Try again tomorrow.')\n const planPages = resolveOrgEntitlements(gate.org).screensPerHost\n const asked = Number(body['maxPages'])\n const maxPages = Math.max(\n 1,\n Math.min(\n SITE_IMPORT_DOOR_MAX_PAGES,\n Number.isFinite(planPages) && planPages > 0 ? planPages : SITE_IMPORT_DOOR_MAX_PAGES,\n Number.isFinite(asked) && asked > 0 ? Math.floor(asked) : SITE_IMPORT_DOOR_MAX_PAGES,\n ),\n )\n const inventory = await crawlSiteForImport(\n { domain, maxPages, timeBudgetMs: SITE_IMPORT_DOOR_TIME_MS, signal: request.signal },\n seams.crawl,\n )\n await audit(gate, 'site-import.crawl', domain, {\n pages: inventory.pages.length,\n skipped: inventory.skipped.length + inventory.skippedOverflow,\n loginPagesRefused: inventory.skipped.filter((entry) => entry.reason === 'login-form').length,\n stoppedBy: inventory.stoppedBy,\n refused: inventory.refused,\n bytesRead: inventory.bytesRead,\n })\n return Response.json({ inventory })\n}\n"],"names":["resolveEffectivePlan","resolveOrgEntitlements","addAdminAudit","checkDomainOwnership","confirmDomainOwnershipCode","DOMAIN_OWNERSHIP_META_NAME","isDomainOwnershipProven","issueDomainOwnershipClaim","listDomainOwnershipClaims","normalizeOwnershipDomain","sendDomainOwnershipCode","emailUnverifiedResponse","firebaseAdmin","isImpersonationSession","invalidIdTokenResponse","lockdownRefusal","getOrgForUser","consumeRateLimit","isServerReleaseFlagOnForOrg","FieldValue","crawlSiteForImport","readSiteVerificationMeta","SITE_IMPORT_RELEASE_FLAG","SITE_IMPORT_DOOR_MAX_PAGES","SITE_IMPORT_CRAWLS_PER_ORG_PER_DAY","SITE_IMPORT_CHECKS_PER_ORG_PER_HOUR","SITE_IMPORT_DOOR_TIME_MS","bearerToken","request","authorization","headers","get","startsWith","slice","length","undefined","siteImportGate","rawOrgId","resolved","idToken","Response","json","error","status","app","decoded","auth","verifyIdToken","unauthenticated","email_verified","staff","orgId","String","trim","uid","member","role","reason","org","locked","firestore","siteImportOfferFor","input","audit","gate","action","domain","after","actorUid","actorEmail","email","target","at","serverTimestamp","console","ownershipDeps","seams","readVerificationMeta","crawl","ownership","tooMany","listSiteImportOwnership","URL","url","searchParams","claims","postSiteImportOwnership","body","catch","deps","claim","method","window","limit","windowMs","allowed","result","detail","provenDetail","sent","address","pendingAddress","postSiteImportCrawl","planPages","screensPerHost","asked","Number","maxPages","Math","max","min","isFinite","floor","inventory","timeBudgetMs","signal","pages","skipped","skippedOverflow","loginPagesRefused","filter","entry","stoppedBy","refused","bytesRead"],"mappings":"AAAA;;;;;;;;;;;;;;;CAeC,GAED;;;;;;;;;;;;;;;;;;;;;;;;;;CA0BC;AAGD,SAASA,oBAAoB,EAAEC,sBAAsB,QAAQ,2CAA0C;AAEvG,SAASC,aAAa,QAAQ,oDAAmD;AACjF,SACEC,oBAAoB,EACpBC,0BAA0B,EAC1BC,0BAA0B,EAC1BC,uBAAuB,EACvBC,yBAAyB,EACzBC,yBAAyB,EACzBC,wBAAwB,EACxBC,uBAAuB,QAElB,mDAAkD;AACzD,SACEC,uBAAuB,EACvBC,aAAa,EACbC,sBAAsB,QACjB,iDAAgD;AACvD,SAASC,sBAAsB,QAAQ,mDAAkD;AACzF,SAASC,eAAe,QAAQ,2CAA0C;AAC1E,SAASC,aAAa,QAAQ,gDAA+C;AAC7E,SAASC,gBAAgB,QAAQ,mDAAkD;AACnF,SAASC,2BAA2B,QAAQ,gDAA+C;AAC3F,SAASC,UAAU,QAAQ,2BAA0B;AACrD,SACEC,kBAAkB,EAClBC,wBAAwB,QAEnB,sCAAkC;AACzC,SAASC,wBAAwB,QAAQ,qCAAiC;AAE1E,SAASA,wBAAwB,GAAE;AAEnC,8EAA8E,GAC9E,OAAO,MAAMC,6BAA6B,GAAE;AAE5C,sDAAsD,GACtD,OAAO,MAAMC,qCAAqC,GAAE;AAEpD,yDAAyD,GACzD,OAAO,MAAMC,sCAAsC,GAAE;AAErD,wEAAwE,GACxE,MAAMC,2BAA2B;AAkBjC,SAASC,YAAYC,OAAgB;QACbA;IAAtB,MAAMC,iBAAgBD,uBAAAA,QAAQE,OAAO,CAACC,GAAG,CAAC,4BAApBH,uBAAwC;IAC9D,OAAOC,cAAcG,UAAU,CAAC,aAAaH,cAAcI,KAAK,CAAC,UAAUC,MAAM,IAAIC;AACvF;AAEA,OAAO,eAAeC,eACpBR,OAAgB,EAChBS,QAAgB;QA0BDC,kBAOHA;IA/BZ,MAAMC,UAAUZ,YAAYC;IAC5B,IAAI,CAACW,SAAS,OAAOC,SAASC,IAAI,CAAC;QAAEC,OAAO;IAAkB,GAAG;QAAEC,QAAQ;IAAI;IAC/E,MAAMC,MAAMhC,cAAcgC,GAAG;IAC7B,IAAIC;IACJ,IAAI;QACFA,UAAU,MAAMD,IAAIE,IAAI,GAAGC,aAAa,CAACR;IAC3C,EAAE,OAAOG,OAAO;QACd,MAAMM,kBAAkBlC,uBAAuB4B;QAC/C,IAAIM,iBAAiB,OAAOA;QAC5B,MAAMN;IACR;IACA,IAAI,CAACG,QAAQI,cAAc,IAAI,CAACpC,uBAAuBgC,UAAU,OAAOlC;IACxE,MAAMuC,QAAQL,OAAO,CAAC,QAAQ,KAAK;IACnC,MAAMM,QAAQC,OAAOf,mBAAAA,WAAY,IAAIgB,IAAI;IACzC,IAAI,CAACF,OAAO;QACV,OAAOX,SAASC,IAAI,CAAC;YAAEC,OAAO;QAA6C,GAAG;YAAEC,QAAQ;QAAI;IAC9F;IACA,MAAML,WAAW,MAAMtB,cAAc6B,QAAQS,GAAG,EAAEH;IAClD,IAAI,CAACb,YAAYA,SAASa,KAAK,KAAKA,OAAO;QACzC,OAAOX,SAASC,IAAI,CAAC;YAAEC,OAAO;QAA4C,GAAG;YAAEC,QAAQ;QAAI;IAC7F;IACA,IAAI,CAACO,SAAS,CAAE,MAAMhC,4BAA4BI,0BAA0B6B,QAAS;QACnF,OAAOX,SAASC,IAAI,CAAC;YAAEC,OAAO;QAAY,GAAG;YAAEC,QAAQ;QAAI;IAC7D;IACA,MAAMY,UAASjB,mBAAAA,SAASiB,MAAM,YAAfjB,mBAAmB;IAClC,IAAI,CAACiB,UAAWA,OAAOC,IAAI,KAAK,WAAWD,OAAOC,IAAI,KAAK,SAAU;QACnE,OAAOhB,SAASC,IAAI,CAClB;YAAEC,OAAO;YAA6De,QAAQ;QAAO,GACrF;YAAEd,QAAQ;QAAI;IAElB;IACA,MAAMe,OAAMpB,gBAAAA,SAASoB,GAAG,YAAZpB,gBAAgB,CAAC;IAC7B,IAAItC,qBAAqB0D,SAAS,QAAQ;QACxC,OAAOlB,SAASC,IAAI,CAClB;YAAEC,OAAO;YAAgEe,QAAQ;QAAO,GACxF;YAAEd,QAAQ;QAAI;IAElB;IACA,MAAMgB,SAAS,MAAM5C,gBAAgB;QACnCa;QACAsB;QACAI,KAAKT,QAAQS,GAAG;QAChBI,KAAKA;IACP;IACA,IAAIC,QAAQ,OAAOA;IACnB,OAAO;QAAEL,KAAKT,QAAQS,GAAG;QAAET;QAASK;QAAOC;QAAOO;QAAKH;QAAQK,WAAWhB,IAAIgB,SAAS;IAAG;AAC5F;AAEA;;;;;;CAMC,GACD,OAAO,eAAeC,mBAAmBC,KAKxC;QAO0BA;IANzB,IAAI;QACF,IAAI,CAACA,MAAMZ,KAAK,IAAI,CAAE,MAAMhC,4BAA4BI,0BAA0BwC,MAAMX,KAAK,GAAI,OAAO;IAC1G,EAAE,eAAM;QACN,OAAO;IACT;IACA,IAAI,CAACW,MAAMP,MAAM,IAAKO,MAAMP,MAAM,CAACC,IAAI,KAAK,WAAWM,MAAMP,MAAM,CAACC,IAAI,KAAK,SAAU,OAAO;IAC9F,IAAIxD,sBAAqB8D,aAAAA,MAAMJ,GAAG,YAATI,aAAa,CAAC,OAAO,QAAQ,OAAO;IAC7D,OAAO;AACT;AAEA,eAAeC,MACbC,IAA2B,EAC3BC,MAAc,EACdC,MAAc,EACdC,KAA8B;IAE9B,IAAI;YAGYH;QAFd,MAAM9D,cAAc8D,KAAKJ,SAAS,EAAE;YAClCQ,UAAUJ,KAAKV,GAAG;YAClBe,UAAU,GAAEL,sBAAAA,KAAKnB,OAAO,CAACyB,KAAK,YAAlBN,sBAAsB;YAClCC;YACAM,QAAQ,CAAC,KAAK,EAAEP,KAAKb,KAAK,CAAC,uBAAuB,EAAEe,QAAQ;YAC5DC,OAAO,aAAKA,OAAWH,KAAKd,KAAK,GAAG;gBAAEA,OAAO;YAAK,IAAI,CAAC;YACvDsB,IAAIrD,WAAWsD,eAAe;QAChC;IACF,EAAE,OAAO/B,OAAO;QACdgC,QAAQhC,KAAK,CAAC,kCAAkC;YAAES,OAAOa,KAAKb,KAAK;YAAEc;YAAQvB;QAAM;IACrF;AACF;AAEA,SAASiC,cAAcC,KAA2B;IAChD,OAAO;QACLC,sBAAsB,CAACX,SACrB7C,yBAAyB6C,QAAQ7D,4BAA4BuE,MAAME,KAAK;OACvEF,MAAMG,SAAS;AAEtB;AAEA,MAAMC,UAAU,CAACtC,QAAkBF,SAASC,IAAI,CAAC;QAAEC;QAAOe,QAAQ;IAAO,GAAG;QAAEd,QAAQ;IAAI;AAE1F,gDAAgD,GAChD,OAAO,eAAesC,wBACpBrD,OAAgB,EAChBgD,QAA8B,CAAC,CAAC;QAElB;IAAd,MAAMzB,SAAQ,oBAAA,IAAI+B,IAAItD,QAAQuD,GAAG,EAAEC,YAAY,CAACrD,GAAG,CAAC,oBAAtC,oBAAkD;IAChE,MAAMiC,OAAO,MAAM5B,eAAeR,SAASuB;IAC3C,IAAIa,gBAAgBxB,UAAU,OAAOwB;IACrC,MAAMqB,SAAS,MAAM7E,0BAA0BwD,KAAKb,KAAK,EAAEwB,cAAcC;IACzE,OAAOpC,SAASC,IAAI,CAAC;QAAE4C;IAAO;AAChC;AAEA,0CAA0C,GAC1C,OAAO,eAAeC,wBACpB1D,OAAgB,EAChBgD,QAA8B,CAAC,CAAC;QAMkBW,aAG5BA,cACAA;IARtB,MAAMA,OAAQ,MAAM3D,QAAQa,IAAI,GAAG+C,KAAK,CAAC,IAAM;IAC/C,IAAI,CAACD,QAAQ,OAAOA,SAAS,UAAU;QACrC,OAAO/C,SAASC,IAAI,CAAC;YAAEC,OAAO;QAAmB,GAAG;YAAEC,QAAQ;QAAI;IACpE;IACA,MAAMqB,OAAO,MAAM5B,eAAeR,SAASwB,QAAOmC,cAAAA,IAAI,CAAC,QAAQ,YAAbA,cAAiB;IACnE,IAAIvB,gBAAgBxB,UAAU,OAAOwB;IACrC,MAAMyB,OAAOd,cAAcC;IAC3B,MAAMX,SAASb,QAAOmC,eAAAA,IAAI,CAAC,SAAS,YAAdA,eAAkB;IACxC,MAAMrB,SAASd,QAAOmC,eAAAA,IAAI,CAAC,SAAS,YAAdA,eAAkB;IAExC,IAAItB,WAAW,SAAS;QACtB,MAAM,EAAEyB,KAAK,EAAEhD,KAAK,EAAE,GAAG,MAAMnC,0BAA0ByD,KAAKb,KAAK,EAAEe,QAAQF,KAAKV,GAAG,EAAEmC;QACvF,IAAI,CAACC,OAAO,OAAOlD,SAASC,IAAI,CAAC;YAAEC;QAAM,GAAG;YAAEC,QAAQ;QAAI;QAC1D,MAAMoB,MAAMC,MAAM,+BAA+B0B,MAAMxB,MAAM,EAAE;YAAEvB,QAAQ+C,MAAM/C,MAAM;QAAC;QACtF,OAAOH,SAASC,IAAI,CAAC;YAAEiD;QAAM;IAC/B;IAEA,IAAIzB,WAAW,SAAS;QACtB,MAAM0B,SAASJ,IAAI,CAAC,SAAS;QAC7B,IAAII,WAAW,SAASA,WAAW,QAAQ;YACzC,OAAOnD,SAASC,IAAI,CAAC;gBAAEC,OAAO;YAA6B,GAAG;gBAAEC,QAAQ;YAAI;QAC9E;QACA,MAAMiD,SAAS,MAAM3E,iBAAiB,CAAC,4BAA4B,EAAE+C,KAAKb,KAAK,EAAE,EAAE;YACjF0C,OAAOpE;YACPqE,UAAU,KAAK,KAAK;QACtB;QACA,IAAI,CAACF,OAAOG,OAAO,EAAE,OAAOf,QAAQ;QACpC,MAAM,EAAEgB,MAAM,EAAEtD,KAAK,EAAE,GAAG,MAAMvC,qBAAqB6D,KAAKb,KAAK,EAAEe,QAAQyB,QAAQF;QACjF,IAAI,CAACO,QAAQ,OAAOxD,SAASC,IAAI,CAAC;YAAEC;QAAM,GAAG;YAAEC,QAAQ;QAAI;QAC3D,IAAIqD,OAAOrD,MAAM,KAAK,UAAU;YAC9B,MAAMoB,MAAMC,MAAM,gCAAgCgC,OAAON,KAAK,CAACxB,MAAM,EAAE;gBACrEyB;gBACAM,QAAQD,OAAON,KAAK,CAACQ,YAAY;YACnC;QACF;QACA,OAAO1D,SAASC,IAAI,CAAC;YAAEuD;QAAO;IAChC;IAEA,IAAI/B,WAAW,aAAa;YAIjBsB;QAHT,MAAMY,OAAO,MAAMzF,wBACjBsD,KAAKb,KAAK,EACVe,QACAd,QAAOmC,cAAAA,IAAI,CAAC,QAAQ,YAAbA,cAAiB,KACxB;YAAE7B,KAAKM,KAAKN,GAAG;QAA4B,GAC3C+B;QAEF,IAAI,CAACU,KAAKT,KAAK,EAAE,OAAOlD,SAASC,IAAI,CAAC;YAAEC,OAAOyD,KAAKzD,KAAK;QAAC,GAAG;YAAEC,QAAQwD,KAAKxD,MAAM;QAAC;QACnF,MAAMoB,MAAMC,MAAM,mCAAmCmC,KAAKT,KAAK,CAACxB,MAAM,EAAE;YACtEkC,SAASD,KAAKT,KAAK,CAACpB,KAAK,CAAC+B,cAAc;QAC1C;QACA,OAAO7D,SAASC,IAAI,CAAC;YAAEiD,OAAOS,KAAKT,KAAK;QAAC;IAC3C;IAEA,IAAIzB,WAAW,gBAAgB;YACyDsB;QAAtF,MAAM,EAAES,MAAM,EAAEtD,KAAK,EAAE,GAAG,MAAMtC,2BAA2B4D,KAAKb,KAAK,EAAEe,QAAQd,QAAOmC,aAAAA,IAAI,CAAC,OAAO,YAAZA,aAAgB,KAAKE;QAC3G,IAAI,CAACO,QAAQ,OAAOxD,SAASC,IAAI,CAAC;YAAEC;QAAM,GAAG;YAAEC,QAAQ;QAAI;QAC3D,IAAIqD,OAAOrD,MAAM,KAAK,UAAU;YAC9B,MAAMoB,MAAMC,MAAM,gCAAgCgC,OAAON,KAAK,CAACxB,MAAM,EAAE;gBACrEyB,QAAQ;gBACRM,QAAQD,OAAON,KAAK,CAACQ,YAAY;YACnC;QACF;QACA,OAAO1D,SAASC,IAAI,CAAC;YAAEuD;QAAO;IAChC;IAEA,OAAOxD,SAASC,IAAI,CAClB;QAAEC,OAAO;IAAyD,GAClE;QAAEC,QAAQ;IAAI;AAElB;AAEA,sCAAsC,GACtC,OAAO,eAAe2D,oBACpB1E,OAAgB,EAChBgD,QAA8B,CAAC,CAAC;QAMkBW,aAEQA;IAN1D,MAAMA,OAAQ,MAAM3D,QAAQa,IAAI,GAAG+C,KAAK,CAAC,IAAM;IAC/C,IAAI,CAACD,QAAQ,OAAOA,SAAS,UAAU;QACrC,OAAO/C,SAASC,IAAI,CAAC;YAAEC,OAAO;QAAmB,GAAG;YAAEC,QAAQ;QAAI;IACpE;IACA,MAAMqB,OAAO,MAAM5B,eAAeR,SAASwB,QAAOmC,cAAAA,IAAI,CAAC,QAAQ,YAAbA,cAAiB;IACnE,IAAIvB,gBAAgBxB,UAAU,OAAOwB;IACrC,MAAM,EAAEE,MAAM,EAAExB,KAAK,EAAE,GAAGjC,yBAAyB2C,QAAOmC,eAAAA,IAAI,CAAC,SAAS,YAAdA,eAAkB;IAC5E,IAAI,CAACrB,QAAQ,OAAO1B,SAASC,IAAI,CAAC;QAAEC;IAAM,GAAG;QAAEC,QAAQ;IAAI;IAC3D,uEAAuE;IACvE,qEAAqE;IACrE,IAAI,CAAE,MAAMrC,wBAAwB0D,KAAKb,KAAK,EAAEe,QAAQS,cAAcC,SAAU;QAC9E,OAAOpC,SAASC,IAAI,CAClB;YAAEC,OAAO;YAAwCe,QAAQ;QAAY,GACrE;YAAEd,QAAQ;QAAI;IAElB;IACA,MAAMiD,SAAS,MAAM3E,iBAAiB,CAAC,kBAAkB,EAAE+C,KAAKb,KAAK,EAAE,EAAE;QACvE0C,OAAOrE;QACPsE,UAAU,KAAK,KAAK,KAAK;IAC3B;IACA,IAAI,CAACF,OAAOG,OAAO,EAAE,OAAOf,QAAQ;IACpC,MAAMuB,YAAYtG,uBAAuB+D,KAAKN,GAAG,EAAE8C,cAAc;IACjE,MAAMC,QAAQC,OAAOnB,IAAI,CAAC,WAAW;IACrC,MAAMoB,WAAWC,KAAKC,GAAG,CACvB,GACAD,KAAKE,GAAG,CACNvF,4BACAmF,OAAOK,QAAQ,CAACR,cAAcA,YAAY,IAAIA,YAAYhF,4BAC1DmF,OAAOK,QAAQ,CAACN,UAAUA,QAAQ,IAAIG,KAAKI,KAAK,CAACP,SAASlF;IAG9D,MAAM0F,YAAY,MAAM7F,mBACtB;QAAE8C;QAAQyC;QAAUO,cAAcxF;QAA0ByF,QAAQvF,QAAQuF,MAAM;IAAC,GACnFvC,MAAME,KAAK;IAEb,MAAMf,MAAMC,MAAM,qBAAqBE,QAAQ;QAC7CkD,OAAOH,UAAUG,KAAK,CAAClF,MAAM;QAC7BmF,SAASJ,UAAUI,OAAO,CAACnF,MAAM,GAAG+E,UAAUK,eAAe;QAC7DC,mBAAmBN,UAAUI,OAAO,CAACG,MAAM,CAAC,CAACC,QAAUA,MAAMhE,MAAM,KAAK,cAAcvB,MAAM;QAC5FwF,WAAWT,UAAUS,SAAS;QAC9BC,SAASV,UAAUU,OAAO;QAC1BC,WAAWX,UAAUW,SAAS;IAChC;IACA,OAAOpF,SAASC,IAAI,CAAC;QAAEwE;IAAU;AACnC"}
|
package/src/lib/server.js
CHANGED
|
@@ -30,6 +30,7 @@ import { registerAiProductsJob } from "./jobs/ai-job-products-step.js";
|
|
|
30
30
|
import { registerAiEmailJob } from "./jobs/ai-job-email-step.js";
|
|
31
31
|
import { registerAiCampaignJob } from "./jobs/ai-job-campaign-step.js";
|
|
32
32
|
import { registerAiSiteJob } from "./jobs/ai-job-site-step.js";
|
|
33
|
+
import { registerAiSiteImportJob } from "./jobs/ai-job-import-step.js";
|
|
33
34
|
import { registerAiBuildJob } from "./jobs/ai-job-build-step.js";
|
|
34
35
|
import { registerAiBuildCapabilities } from "./jobs/ai-build-capabilities.js";
|
|
35
36
|
import { registerAiWorkflowJob } from "./jobs/ai-job-workflow-step.js";
|
|
@@ -73,6 +74,7 @@ import { GET as aiUsage } from "./server/ai-usage.js";
|
|
|
73
74
|
import { GET as aiAllotments } from "./server/ai-allotments.js";
|
|
74
75
|
import { GET as aiInsightAnswer } from "./server/ai-insight-answer.js";
|
|
75
76
|
import { POST as runAiInsightsDigest } from "./server/ai-insight-digest-route.js";
|
|
77
|
+
import { listSiteImportOwnership, postSiteImportCrawl, postSiteImportOwnership } from "./server/ai-site-import-route.js";
|
|
76
78
|
import { AI_INSIGHTS_DIGEST_PATH } from "./insights/ai-insight-digest.js";
|
|
77
79
|
import { GET as aiModels } from "./server/ai-models.js";
|
|
78
80
|
import { GET as billingCredits } from "./server/billing-credits.js";
|
|
@@ -151,6 +153,9 @@ export * from "./runtime/ai-runtime.js";
|
|
|
151
153
|
// The site scaffold, which builds a whole site through the steps above
|
|
152
154
|
// (AGL-2911).
|
|
153
155
|
registerAiSiteJob();
|
|
156
|
+
// Bring my site (AGL-3976): a proved website brought in as draft pages, in
|
|
157
|
+
// waves a person starts one at a time, through the page and layout steps.
|
|
158
|
+
registerAiSiteImportJob();
|
|
154
159
|
// A build from one request (AGL-3616): pages, creations and other
|
|
155
160
|
// plugins' drafts, through the steps above and the writers those plugins
|
|
156
161
|
// register, settled item by item. What this plugin lets a build make is
|
|
@@ -303,6 +308,15 @@ export * from "./runtime/ai-runtime.js";
|
|
|
303
308
|
registerPluginApiRoute('ai/media/images', {
|
|
304
309
|
web: createAiMediaImages
|
|
305
310
|
});
|
|
311
|
+
// "Bring my site" (AGL-3974): proving the workspace owns a website, and
|
|
312
|
+
// reading it once it has. Behind `release_ai_site_import`, paid plans,
|
|
313
|
+
// owners and admins; the crawl refuses a domain without a current proof.
|
|
314
|
+
registerPluginApiRoute('ai/site-import/ownership', {
|
|
315
|
+
web: (request)=>request.method === 'GET' ? listSiteImportOwnership(request) : postSiteImportOwnership(request)
|
|
316
|
+
});
|
|
317
|
+
registerPluginApiRoute('ai/site-import/crawl', {
|
|
318
|
+
web: (request)=>postSiteImportCrawl(request)
|
|
319
|
+
});
|
|
306
320
|
registerPluginApiRoute('ai/billing/credits', {
|
|
307
321
|
web: billingCredits
|
|
308
322
|
});
|