@aglyn/tenant-data-admin 1.0.0-beta.256 → 1.0.0-beta.258
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +7 -7
- package/src/lib/server/media-cdn-icon.d.ts +2 -2
- package/src/lib/server/media-cdn-icon.js +4 -1
- package/src/lib/server/media-cdn-icon.js.map +1 -1
- package/src/lib/server/media-delivery-regeneration.d.ts +2 -2
- package/src/lib/server/media-delivery-regeneration.js +4 -1
- package/src/lib/server/media-delivery-regeneration.js.map +1 -1
- package/src/lib/server/personal-data-export.js +32 -2
- package/src/lib/server/personal-data-export.js.map +1 -1
- package/src/lib/server/storage-read-stream.js +4 -0
- package/src/lib/server/storage-read-stream.js.map +1 -1
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@aglyn/tenant-data-admin",
|
|
3
|
-
"version": "1.0.0-beta.
|
|
3
|
+
"version": "1.0.0-beta.258",
|
|
4
4
|
"license": "Apache-2.0",
|
|
5
5
|
"homepage": "https://aglyn.com",
|
|
6
6
|
"repository": {
|
|
@@ -25,12 +25,12 @@
|
|
|
25
25
|
"./package.json": "./package.json"
|
|
26
26
|
},
|
|
27
27
|
"dependencies": {
|
|
28
|
-
"@aglyn/aglyn": "1.0.0-beta.
|
|
29
|
-
"@aglyn/shared-util-email": "1.0.0-beta.
|
|
30
|
-
"@aglyn/shared-util-fbserver": "1.0.0-beta.
|
|
31
|
-
"@aglyn/shared-util-first-touch": "1.0.0-beta.
|
|
32
|
-
"@aglyn/shared-util-http": "1.0.0-beta.
|
|
33
|
-
"@aglyn/shared-util-tools": "1.0.0-beta.
|
|
28
|
+
"@aglyn/aglyn": "1.0.0-beta.258",
|
|
29
|
+
"@aglyn/shared-util-email": "1.0.0-beta.258",
|
|
30
|
+
"@aglyn/shared-util-fbserver": "1.0.0-beta.258",
|
|
31
|
+
"@aglyn/shared-util-first-touch": "1.0.0-beta.258",
|
|
32
|
+
"@aglyn/shared-util-http": "1.0.0-beta.258",
|
|
33
|
+
"@aglyn/shared-util-tools": "1.0.0-beta.258",
|
|
34
34
|
"@msgpack/msgpack": "^3.1.3",
|
|
35
35
|
"@swc/helpers": "0.5.23",
|
|
36
36
|
"sharp": "^0.35.5",
|
|
@@ -17,6 +17,7 @@
|
|
|
17
17
|
import { type SiteIconSpec } from '@aglyn/aglyn/app-utils/site-icon-set';
|
|
18
18
|
import type { NextApiRequest, NextApiResponse } from 'next';
|
|
19
19
|
import type { MediaCdnRateLimitRefusal } from './media-cdn-rate-limit';
|
|
20
|
+
import { type StorageReadableFile } from './storage-read-stream';
|
|
20
21
|
/**
|
|
21
22
|
* The `?icon=` representation of a media CDN URL (AGL-3484): one icon of a
|
|
22
23
|
* site's derived set, drawn from the asset's current bytes.
|
|
@@ -52,11 +53,10 @@ export interface MediaCdnIconSource {
|
|
|
52
53
|
/** `/api/media/cdn/{scope}/{mediaId}` — the asset's stable URL. */
|
|
53
54
|
stablePath: string;
|
|
54
55
|
/** The Storage object, as far as this reads it. */
|
|
55
|
-
file: {
|
|
56
|
+
file: StorageReadableFile & {
|
|
56
57
|
getMetadata(): Promise<[{
|
|
57
58
|
size?: string | number;
|
|
58
59
|
} | null | undefined]>;
|
|
59
|
-
download(): Promise<[Buffer]>;
|
|
60
60
|
};
|
|
61
61
|
}
|
|
62
62
|
/** The `ETag` of one derived icon: the source's hash plus the representation. */
|
|
@@ -16,6 +16,7 @@
|
|
|
16
16
|
*/ import { formatSiteIconSpec, normalizeSiteIconBackground, SITE_ICON_BACKGROUND_PARAM, SITE_ICON_PARAM, SITE_ICON_VERSION_PARAM, siteIconSourceTypeDerivable, siteIconUsesBackground, DEFAULT_SITE_ICON_BACKGROUND } from "@aglyn/aglyn/app-utils/site-icon-set";
|
|
17
17
|
import { MEDIA_VARIANT_SOURCE_MAX_BYTES } from "./media-variants.js";
|
|
18
18
|
import { renderSiteIcon } from "./site-icon-render.js";
|
|
19
|
+
import { readStorageObject } from "./storage-read-stream.js";
|
|
19
20
|
const firstValue = (value)=>{
|
|
20
21
|
const raw = Array.isArray(value) ? value[0] : value;
|
|
21
22
|
return typeof raw === 'string' && raw !== '' ? raw : undefined;
|
|
@@ -114,7 +115,9 @@ export async function serveMediaCdnIcon(options) {
|
|
|
114
115
|
}
|
|
115
116
|
let icon;
|
|
116
117
|
try {
|
|
117
|
-
|
|
118
|
+
// Not `file.download()`: that is the same read without the listener
|
|
119
|
+
// budget, so every rendered icon would log a false leak warning.
|
|
120
|
+
const bytes = await readStorageObject(source.file);
|
|
118
121
|
icon = await renderSiteIcon({
|
|
119
122
|
source: bytes,
|
|
120
123
|
spec,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../../../../../../../../libs/tenant/data/admin/src/lib/server/media-cdn-icon.ts"],"sourcesContent":["/**\n * @license\n * Copyright 2026 Aglyn LLC\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport {\n formatSiteIconSpec,\n normalizeSiteIconBackground,\n SITE_ICON_BACKGROUND_PARAM,\n SITE_ICON_PARAM,\n SITE_ICON_VERSION_PARAM,\n siteIconSourceTypeDerivable,\n siteIconUsesBackground,\n DEFAULT_SITE_ICON_BACKGROUND,\n type SiteIconSpec,\n} from '@aglyn/aglyn/app-utils/site-icon-set'\nimport type { NextApiRequest, NextApiResponse } from 'next'\n\nimport type { MediaCdnRateLimitRefusal } from './media-cdn-rate-limit'\nimport { MEDIA_VARIANT_SOURCE_MAX_BYTES } from './media-variants'\nimport { renderSiteIcon } from './site-icon-render'\n\n/**\n * The `?icon=` representation of a media CDN URL (AGL-3484): one icon of a\n * site's derived set, drawn from the asset's current bytes.\n *\n * `serveMediaCdn` calls this only after every gate it applies to the bytes —\n * scope, lockdown, quarantine, the private signature and the stale-pin\n * redirect — so an icon is served exactly where its source would be.\n *\n * ## Derived once, cached by version\n *\n * The URL a page names carries `v`, the asset's `contentHash`. While `v` is\n * current the icon is drawn once and held for a year by the edge and the\n * browser (`immutable`): the same URL can never need different bytes, because\n * a DAM Replace changes the hash and so the URL the next render names. A\n * stale `v` 302s to the current one under the stable URL's revalidated policy,\n * which is how a page rendered before a Replace reaches the new icon. A URL\n * with no `v` (the source's hash is unknown) is served under the stable\n * policy with an ETag keyed on the hash, exactly as the asset itself is.\n *\n * ## Anything the renderer cannot draw falls back to the original\n *\n * A non-image, an ICO (which `sharp` cannot decode), a source over the\n * variant generator's fetch ceiling, or a file that fails to decode: each\n * 302s to the asset's plain URL. A favicon link must never be the reason a\n * tab shows nothing.\n */\n\n/** The fields of a stored asset this representation reads. */\nexport interface MediaCdnIconSource {\n /** The media document's `contentType`. */\n contentType: unknown\n /** The media document's `contentHash`; `''` for a legacy asset. */\n contentHash: string\n /** `/api/media/cdn/{scope}/{mediaId}` — the asset's stable URL. */\n stablePath: string\n /** The Storage object, as far as this reads it. */\n file: {\n getMetadata(): Promise<[{ size?: string | number } | null | undefined]>\n download(): Promise<[Buffer]>\n }\n}\n\nconst firstValue = (value: unknown): string | undefined => {\n const raw = Array.isArray(value) ? value[0] : value\n return typeof raw === 'string' && raw !== '' ? raw : undefined\n}\n\n/**\n * The query this representation names, rebuilt from the parameters it reads —\n * never forwarded whole, for `mediaCdnForwardedQuery`'s reasons. `exp`/`sig`\n * ride along so a signed asset's redirect still serves.\n */\nfunction iconQuery(\n query: NextApiRequest['query'],\n pairs: Array<[string, string | undefined]>,\n): string {\n const params = new URLSearchParams()\n for (const [key, value] of pairs) if (value) params.set(key, value)\n for (const key of ['exp', 'sig']) {\n const value = firstValue(query[key])\n if (value) params.set(key, value)\n }\n const encoded = params.toString()\n return encoded ? `?${encoded}` : ''\n}\n\n/** The `ETag` of one derived icon: the source's hash plus the representation. */\nexport function mediaCdnIconEtag(\n contentHash: string,\n spec: SiteIconSpec,\n background: string | undefined,\n): string | null {\n if (!contentHash) return null\n const plate = background ? `-${background}` : ''\n return `\"${contentHash}-icon-${formatSiteIconSpec(spec)}${plate}\"`\n}\n\nexport async function serveMediaCdnIcon(options: {\n req: NextApiRequest\n res: NextApiResponse\n spec: SiteIconSpec\n source: MediaCdnIconSource\n /** Forces `private, no-store` on a signed asset, as the handler's does. */\n setCacheControl: (value: string) => void\n cacheControl: { stable: string; immutable: string }\n /** The caller's delivery count; `null` means serve. */\n rateLimit: () => Promise<MediaCdnRateLimitRefusal | null>\n}): Promise<void> {\n const { req, res, spec, source, setCacheControl, cacheControl } = options\n const background = siteIconUsesBackground(spec)\n ? (normalizeSiteIconBackground(\n `#${firstValue(req.query[SITE_ICON_BACKGROUND_PARAM]) ?? ''}`,\n ) ?? DEFAULT_SITE_ICON_BACKGROUND)\n : undefined\n const version = firstValue(req.query[SITE_ICON_VERSION_PARAM])\n const { contentHash } = source\n\n const original = (cache: string) => {\n res.removeHeader('ETag')\n setCacheControl(cache)\n res.setHeader('Location', `${source.stablePath}${iconQuery(req.query, [])}`)\n res.status(302).end()\n }\n\n if (!siteIconSourceTypeDerivable(source.contentType)) {\n original(cacheControl.stable)\n return\n }\n if (version && contentHash && version !== contentHash) {\n setCacheControl(cacheControl.stable)\n res.setHeader(\n 'Location',\n `${source.stablePath}${iconQuery(req.query, [\n [SITE_ICON_PARAM, formatSiteIconSpec(spec)],\n [SITE_ICON_BACKGROUND_PARAM, background],\n [SITE_ICON_VERSION_PARAM, contentHash],\n ])}`,\n )\n res.status(302).end()\n return\n }\n\n const etag = mediaCdnIconEtag(contentHash, spec, background)\n setCacheControl(\n version && version === contentHash\n ? cacheControl.immutable\n : cacheControl.stable,\n )\n if (etag) {\n res.setHeader('ETag', etag)\n if (req.headers['if-none-match'] === etag) {\n res.status(304).end()\n return\n }\n }\n\n const counted = req.method === 'GET' ? options.rateLimit() : null\n const [metadata] = await source.file.getMetadata().catch(() => [null])\n if (!metadata) {\n res.removeHeader('ETag')\n setCacheControl('public, max-age=60')\n res.status(404).json({ error: 'Not found' })\n return\n }\n if (Number(metadata.size ?? 0) > MEDIA_VARIANT_SOURCE_MAX_BYTES) {\n original(cacheControl.stable)\n return\n }\n const refused = await counted\n if (refused) {\n res.removeHeader('ETag')\n res.setHeader('Cache-Control', 'no-store')\n res.setHeader('Retry-After', String(refused.retryAfterSeconds))\n res.status(429).json({ error: 'Too many requests' })\n return\n }\n\n let icon: { body: Buffer; contentType: string }\n try {\n const [bytes] = await source.file.download()\n icon = await renderSiteIcon({ source: bytes, spec, background })\n } catch (error) {\n // A file that will not decode is a property of the upload, not of the\n // request, but it is not cached either: a Replace must be able to fix it.\n console.error('[media-cdn] site icon render failed', source.stablePath, error)\n original('no-store')\n return\n }\n\n res.setHeader('Content-Type', icon.contentType)\n res.setHeader('Content-Length', String(icon.body.length))\n res.setHeader(\n 'Content-Disposition',\n `inline; filename=\"icon-${formatSiteIconSpec(spec)}.${\n spec.plate === 'ico' ? 'ico' : 'png'\n }\"`,\n )\n if (req.method === 'HEAD') {\n res.status(200).end()\n return\n }\n res.status(200).end(icon.body)\n}\n"],"names":["formatSiteIconSpec","normalizeSiteIconBackground","SITE_ICON_BACKGROUND_PARAM","SITE_ICON_PARAM","SITE_ICON_VERSION_PARAM","siteIconSourceTypeDerivable","siteIconUsesBackground","DEFAULT_SITE_ICON_BACKGROUND","MEDIA_VARIANT_SOURCE_MAX_BYTES","renderSiteIcon","firstValue","value","raw","Array","isArray","undefined","iconQuery","query","pairs","params","URLSearchParams","key","set","encoded","toString","mediaCdnIconEtag","contentHash","spec","background","plate","serveMediaCdnIcon","options","metadata","req","res","source","setCacheControl","cacheControl","version","original","cache","removeHeader","setHeader","stablePath","status","end","contentType","stable","etag","immutable","headers","counted","method","rateLimit","file","getMetadata","catch","json","error","Number","size","refused","String","retryAfterSeconds","icon","bytes","download","console","body","length"],"mappings":"AAAA;;;;;;;;;;;;;;;CAeC,GAED,SACEA,kBAAkB,EAClBC,2BAA2B,EAC3BC,0BAA0B,EAC1BC,eAAe,EACfC,uBAAuB,EACvBC,2BAA2B,EAC3BC,sBAAsB,EACtBC,4BAA4B,QAEvB,uCAAsC;AAI7C,SAASC,8BAA8B,QAAQ,sBAAkB;AACjE,SAASC,cAAc,QAAQ,wBAAoB;AA4CnD,MAAMC,aAAa,CAACC;IAClB,MAAMC,MAAMC,MAAMC,OAAO,CAACH,SAASA,KAAK,CAAC,EAAE,GAAGA;IAC9C,OAAO,OAAOC,QAAQ,YAAYA,QAAQ,KAAKA,MAAMG;AACvD;AAEA;;;;CAIC,GACD,SAASC,UACPC,KAA8B,EAC9BC,KAA0C;IAE1C,MAAMC,SAAS,IAAIC;IACnB,KAAK,MAAM,CAACC,KAAKV,MAAM,IAAIO,MAAO,IAAIP,OAAOQ,OAAOG,GAAG,CAACD,KAAKV;IAC7D,KAAK,MAAMU,OAAO;QAAC;QAAO;KAAM,CAAE;QAChC,MAAMV,QAAQD,WAAWO,KAAK,CAACI,IAAI;QACnC,IAAIV,OAAOQ,OAAOG,GAAG,CAACD,KAAKV;IAC7B;IACA,MAAMY,UAAUJ,OAAOK,QAAQ;IAC/B,OAAOD,UAAU,CAAC,CAAC,EAAEA,SAAS,GAAG;AACnC;AAEA,+EAA+E,GAC/E,OAAO,SAASE,iBACdC,WAAmB,EACnBC,IAAkB,EAClBC,UAA8B;IAE9B,IAAI,CAACF,aAAa,OAAO;IACzB,MAAMG,QAAQD,aAAa,CAAC,CAAC,EAAEA,YAAY,GAAG;IAC9C,OAAO,CAAC,CAAC,EAAEF,YAAY,MAAM,EAAE1B,mBAAmB2B,QAAQE,MAAM,CAAC,CAAC;AACpE;AAEA,OAAO,eAAeC,kBAAkBC,OAUvC;QAGM9B,8BACKS,aAqDCsB;IAxDX,MAAM,EAAEC,GAAG,EAAEC,GAAG,EAAEP,IAAI,EAAEQ,MAAM,EAAEC,eAAe,EAAEC,YAAY,EAAE,GAAGN;IAClE,MAAMH,aAAatB,uBAAuBqB,SACrC1B,+BAAAA,4BACC,CAAC,CAAC,GAAES,cAAAA,WAAWuB,IAAIhB,KAAK,CAACf,2BAA2B,aAAhDQ,cAAqD,IAAI,aAD9DT,+BAEIM,+BACLQ;IACJ,MAAMuB,UAAU5B,WAAWuB,IAAIhB,KAAK,CAACb,wBAAwB;IAC7D,MAAM,EAAEsB,WAAW,EAAE,GAAGS;IAExB,MAAMI,WAAW,CAACC;QAChBN,IAAIO,YAAY,CAAC;QACjBL,gBAAgBI;QAChBN,IAAIQ,SAAS,CAAC,YAAY,GAAGP,OAAOQ,UAAU,GAAG3B,UAAUiB,IAAIhB,KAAK,EAAE,EAAE,GAAG;QAC3EiB,IAAIU,MAAM,CAAC,KAAKC,GAAG;IACrB;IAEA,IAAI,CAACxC,4BAA4B8B,OAAOW,WAAW,GAAG;QACpDP,SAASF,aAAaU,MAAM;QAC5B;IACF;IACA,IAAIT,WAAWZ,eAAeY,YAAYZ,aAAa;QACrDU,gBAAgBC,aAAaU,MAAM;QACnCb,IAAIQ,SAAS,CACX,YACA,GAAGP,OAAOQ,UAAU,GAAG3B,UAAUiB,IAAIhB,KAAK,EAAE;YAC1C;gBAACd;gBAAiBH,mBAAmB2B;aAAM;YAC3C;gBAACzB;gBAA4B0B;aAAW;YACxC;gBAACxB;gBAAyBsB;aAAY;SACvC,GAAG;QAENQ,IAAIU,MAAM,CAAC,KAAKC,GAAG;QACnB;IACF;IAEA,MAAMG,OAAOvB,iBAAiBC,aAAaC,MAAMC;IACjDQ,gBACEE,WAAWA,YAAYZ,cACnBW,aAAaY,SAAS,GACtBZ,aAAaU,MAAM;IAEzB,IAAIC,MAAM;QACRd,IAAIQ,SAAS,CAAC,QAAQM;QACtB,IAAIf,IAAIiB,OAAO,CAAC,gBAAgB,KAAKF,MAAM;YACzCd,IAAIU,MAAM,CAAC,KAAKC,GAAG;YACnB;QACF;IACF;IAEA,MAAMM,UAAUlB,IAAImB,MAAM,KAAK,QAAQrB,QAAQsB,SAAS,KAAK;IAC7D,MAAM,CAACrB,SAAS,GAAG,MAAMG,OAAOmB,IAAI,CAACC,WAAW,GAAGC,KAAK,CAAC,IAAM;YAAC;SAAK;IACrE,IAAI,CAACxB,UAAU;QACbE,IAAIO,YAAY,CAAC;QACjBL,gBAAgB;QAChBF,IAAIU,MAAM,CAAC,KAAKa,IAAI,CAAC;YAAEC,OAAO;QAAY;QAC1C;IACF;IACA,IAAIC,QAAO3B,iBAAAA,SAAS4B,IAAI,YAAb5B,iBAAiB,KAAKxB,gCAAgC;QAC/D+B,SAASF,aAAaU,MAAM;QAC5B;IACF;IACA,MAAMc,UAAU,MAAMV;IACtB,IAAIU,SAAS;QACX3B,IAAIO,YAAY,CAAC;QACjBP,IAAIQ,SAAS,CAAC,iBAAiB;QAC/BR,IAAIQ,SAAS,CAAC,eAAeoB,OAAOD,QAAQE,iBAAiB;QAC7D7B,IAAIU,MAAM,CAAC,KAAKa,IAAI,CAAC;YAAEC,OAAO;QAAoB;QAClD;IACF;IAEA,IAAIM;IACJ,IAAI;QACF,MAAM,CAACC,MAAM,GAAG,MAAM9B,OAAOmB,IAAI,CAACY,QAAQ;QAC1CF,OAAO,MAAMvD,eAAe;YAAE0B,QAAQ8B;YAAOtC;YAAMC;QAAW;IAChE,EAAE,OAAO8B,OAAO;QACd,sEAAsE;QACtE,0EAA0E;QAC1ES,QAAQT,KAAK,CAAC,uCAAuCvB,OAAOQ,UAAU,EAAEe;QACxEnB,SAAS;QACT;IACF;IAEAL,IAAIQ,SAAS,CAAC,gBAAgBsB,KAAKlB,WAAW;IAC9CZ,IAAIQ,SAAS,CAAC,kBAAkBoB,OAAOE,KAAKI,IAAI,CAACC,MAAM;IACvDnC,IAAIQ,SAAS,CACX,uBACA,CAAC,uBAAuB,EAAE1C,mBAAmB2B,MAAM,CAAC,EAClDA,KAAKE,KAAK,KAAK,QAAQ,QAAQ,MAChC,CAAC,CAAC;IAEL,IAAII,IAAImB,MAAM,KAAK,QAAQ;QACzBlB,IAAIU,MAAM,CAAC,KAAKC,GAAG;QACnB;IACF;IACAX,IAAIU,MAAM,CAAC,KAAKC,GAAG,CAACmB,KAAKI,IAAI;AAC/B"}
|
|
1
|
+
{"version":3,"sources":["../../../../../../../../libs/tenant/data/admin/src/lib/server/media-cdn-icon.ts"],"sourcesContent":["/**\n * @license\n * Copyright 2026 Aglyn LLC\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport {\n formatSiteIconSpec,\n normalizeSiteIconBackground,\n SITE_ICON_BACKGROUND_PARAM,\n SITE_ICON_PARAM,\n SITE_ICON_VERSION_PARAM,\n siteIconSourceTypeDerivable,\n siteIconUsesBackground,\n DEFAULT_SITE_ICON_BACKGROUND,\n type SiteIconSpec,\n} from '@aglyn/aglyn/app-utils/site-icon-set'\nimport type { NextApiRequest, NextApiResponse } from 'next'\n\nimport type { MediaCdnRateLimitRefusal } from './media-cdn-rate-limit'\nimport { MEDIA_VARIANT_SOURCE_MAX_BYTES } from './media-variants'\nimport { renderSiteIcon } from './site-icon-render'\nimport { readStorageObject, type StorageReadableFile } from './storage-read-stream'\n\n/**\n * The `?icon=` representation of a media CDN URL (AGL-3484): one icon of a\n * site's derived set, drawn from the asset's current bytes.\n *\n * `serveMediaCdn` calls this only after every gate it applies to the bytes —\n * scope, lockdown, quarantine, the private signature and the stale-pin\n * redirect — so an icon is served exactly where its source would be.\n *\n * ## Derived once, cached by version\n *\n * The URL a page names carries `v`, the asset's `contentHash`. While `v` is\n * current the icon is drawn once and held for a year by the edge and the\n * browser (`immutable`): the same URL can never need different bytes, because\n * a DAM Replace changes the hash and so the URL the next render names. A\n * stale `v` 302s to the current one under the stable URL's revalidated policy,\n * which is how a page rendered before a Replace reaches the new icon. A URL\n * with no `v` (the source's hash is unknown) is served under the stable\n * policy with an ETag keyed on the hash, exactly as the asset itself is.\n *\n * ## Anything the renderer cannot draw falls back to the original\n *\n * A non-image, an ICO (which `sharp` cannot decode), a source over the\n * variant generator's fetch ceiling, or a file that fails to decode: each\n * 302s to the asset's plain URL. A favicon link must never be the reason a\n * tab shows nothing.\n */\n\n/** The fields of a stored asset this representation reads. */\nexport interface MediaCdnIconSource {\n /** The media document's `contentType`. */\n contentType: unknown\n /** The media document's `contentHash`; `''` for a legacy asset. */\n contentHash: string\n /** `/api/media/cdn/{scope}/{mediaId}` — the asset's stable URL. */\n stablePath: string\n /** The Storage object, as far as this reads it. */\n file: StorageReadableFile & {\n getMetadata(): Promise<[{ size?: string | number } | null | undefined]>\n }\n}\n\nconst firstValue = (value: unknown): string | undefined => {\n const raw = Array.isArray(value) ? value[0] : value\n return typeof raw === 'string' && raw !== '' ? raw : undefined\n}\n\n/**\n * The query this representation names, rebuilt from the parameters it reads —\n * never forwarded whole, for `mediaCdnForwardedQuery`'s reasons. `exp`/`sig`\n * ride along so a signed asset's redirect still serves.\n */\nfunction iconQuery(\n query: NextApiRequest['query'],\n pairs: Array<[string, string | undefined]>,\n): string {\n const params = new URLSearchParams()\n for (const [key, value] of pairs) if (value) params.set(key, value)\n for (const key of ['exp', 'sig']) {\n const value = firstValue(query[key])\n if (value) params.set(key, value)\n }\n const encoded = params.toString()\n return encoded ? `?${encoded}` : ''\n}\n\n/** The `ETag` of one derived icon: the source's hash plus the representation. */\nexport function mediaCdnIconEtag(\n contentHash: string,\n spec: SiteIconSpec,\n background: string | undefined,\n): string | null {\n if (!contentHash) return null\n const plate = background ? `-${background}` : ''\n return `\"${contentHash}-icon-${formatSiteIconSpec(spec)}${plate}\"`\n}\n\nexport async function serveMediaCdnIcon(options: {\n req: NextApiRequest\n res: NextApiResponse\n spec: SiteIconSpec\n source: MediaCdnIconSource\n /** Forces `private, no-store` on a signed asset, as the handler's does. */\n setCacheControl: (value: string) => void\n cacheControl: { stable: string; immutable: string }\n /** The caller's delivery count; `null` means serve. */\n rateLimit: () => Promise<MediaCdnRateLimitRefusal | null>\n}): Promise<void> {\n const { req, res, spec, source, setCacheControl, cacheControl } = options\n const background = siteIconUsesBackground(spec)\n ? (normalizeSiteIconBackground(\n `#${firstValue(req.query[SITE_ICON_BACKGROUND_PARAM]) ?? ''}`,\n ) ?? DEFAULT_SITE_ICON_BACKGROUND)\n : undefined\n const version = firstValue(req.query[SITE_ICON_VERSION_PARAM])\n const { contentHash } = source\n\n const original = (cache: string) => {\n res.removeHeader('ETag')\n setCacheControl(cache)\n res.setHeader('Location', `${source.stablePath}${iconQuery(req.query, [])}`)\n res.status(302).end()\n }\n\n if (!siteIconSourceTypeDerivable(source.contentType)) {\n original(cacheControl.stable)\n return\n }\n if (version && contentHash && version !== contentHash) {\n setCacheControl(cacheControl.stable)\n res.setHeader(\n 'Location',\n `${source.stablePath}${iconQuery(req.query, [\n [SITE_ICON_PARAM, formatSiteIconSpec(spec)],\n [SITE_ICON_BACKGROUND_PARAM, background],\n [SITE_ICON_VERSION_PARAM, contentHash],\n ])}`,\n )\n res.status(302).end()\n return\n }\n\n const etag = mediaCdnIconEtag(contentHash, spec, background)\n setCacheControl(\n version && version === contentHash\n ? cacheControl.immutable\n : cacheControl.stable,\n )\n if (etag) {\n res.setHeader('ETag', etag)\n if (req.headers['if-none-match'] === etag) {\n res.status(304).end()\n return\n }\n }\n\n const counted = req.method === 'GET' ? options.rateLimit() : null\n const [metadata] = await source.file.getMetadata().catch(() => [null])\n if (!metadata) {\n res.removeHeader('ETag')\n setCacheControl('public, max-age=60')\n res.status(404).json({ error: 'Not found' })\n return\n }\n if (Number(metadata.size ?? 0) > MEDIA_VARIANT_SOURCE_MAX_BYTES) {\n original(cacheControl.stable)\n return\n }\n const refused = await counted\n if (refused) {\n res.removeHeader('ETag')\n res.setHeader('Cache-Control', 'no-store')\n res.setHeader('Retry-After', String(refused.retryAfterSeconds))\n res.status(429).json({ error: 'Too many requests' })\n return\n }\n\n let icon: { body: Buffer; contentType: string }\n try {\n // Not `file.download()`: that is the same read without the listener\n // budget, so every rendered icon would log a false leak warning.\n const bytes = await readStorageObject(source.file)\n icon = await renderSiteIcon({ source: bytes, spec, background })\n } catch (error) {\n // A file that will not decode is a property of the upload, not of the\n // request, but it is not cached either: a Replace must be able to fix it.\n console.error('[media-cdn] site icon render failed', source.stablePath, error)\n original('no-store')\n return\n }\n\n res.setHeader('Content-Type', icon.contentType)\n res.setHeader('Content-Length', String(icon.body.length))\n res.setHeader(\n 'Content-Disposition',\n `inline; filename=\"icon-${formatSiteIconSpec(spec)}.${\n spec.plate === 'ico' ? 'ico' : 'png'\n }\"`,\n )\n if (req.method === 'HEAD') {\n res.status(200).end()\n return\n }\n res.status(200).end(icon.body)\n}\n"],"names":["formatSiteIconSpec","normalizeSiteIconBackground","SITE_ICON_BACKGROUND_PARAM","SITE_ICON_PARAM","SITE_ICON_VERSION_PARAM","siteIconSourceTypeDerivable","siteIconUsesBackground","DEFAULT_SITE_ICON_BACKGROUND","MEDIA_VARIANT_SOURCE_MAX_BYTES","renderSiteIcon","readStorageObject","firstValue","value","raw","Array","isArray","undefined","iconQuery","query","pairs","params","URLSearchParams","key","set","encoded","toString","mediaCdnIconEtag","contentHash","spec","background","plate","serveMediaCdnIcon","options","metadata","req","res","source","setCacheControl","cacheControl","version","original","cache","removeHeader","setHeader","stablePath","status","end","contentType","stable","etag","immutable","headers","counted","method","rateLimit","file","getMetadata","catch","json","error","Number","size","refused","String","retryAfterSeconds","icon","bytes","console","body","length"],"mappings":"AAAA;;;;;;;;;;;;;;;CAeC,GAED,SACEA,kBAAkB,EAClBC,2BAA2B,EAC3BC,0BAA0B,EAC1BC,eAAe,EACfC,uBAAuB,EACvBC,2BAA2B,EAC3BC,sBAAsB,EACtBC,4BAA4B,QAEvB,uCAAsC;AAI7C,SAASC,8BAA8B,QAAQ,sBAAkB;AACjE,SAASC,cAAc,QAAQ,wBAAoB;AACnD,SAASC,iBAAiB,QAAkC,2BAAuB;AA2CnF,MAAMC,aAAa,CAACC;IAClB,MAAMC,MAAMC,MAAMC,OAAO,CAACH,SAASA,KAAK,CAAC,EAAE,GAAGA;IAC9C,OAAO,OAAOC,QAAQ,YAAYA,QAAQ,KAAKA,MAAMG;AACvD;AAEA;;;;CAIC,GACD,SAASC,UACPC,KAA8B,EAC9BC,KAA0C;IAE1C,MAAMC,SAAS,IAAIC;IACnB,KAAK,MAAM,CAACC,KAAKV,MAAM,IAAIO,MAAO,IAAIP,OAAOQ,OAAOG,GAAG,CAACD,KAAKV;IAC7D,KAAK,MAAMU,OAAO;QAAC;QAAO;KAAM,CAAE;QAChC,MAAMV,QAAQD,WAAWO,KAAK,CAACI,IAAI;QACnC,IAAIV,OAAOQ,OAAOG,GAAG,CAACD,KAAKV;IAC7B;IACA,MAAMY,UAAUJ,OAAOK,QAAQ;IAC/B,OAAOD,UAAU,CAAC,CAAC,EAAEA,SAAS,GAAG;AACnC;AAEA,+EAA+E,GAC/E,OAAO,SAASE,iBACdC,WAAmB,EACnBC,IAAkB,EAClBC,UAA8B;IAE9B,IAAI,CAACF,aAAa,OAAO;IACzB,MAAMG,QAAQD,aAAa,CAAC,CAAC,EAAEA,YAAY,GAAG;IAC9C,OAAO,CAAC,CAAC,EAAEF,YAAY,MAAM,EAAE3B,mBAAmB4B,QAAQE,MAAM,CAAC,CAAC;AACpE;AAEA,OAAO,eAAeC,kBAAkBC,OAUvC;QAGM/B,8BACKU,aAqDCsB;IAxDX,MAAM,EAAEC,GAAG,EAAEC,GAAG,EAAEP,IAAI,EAAEQ,MAAM,EAAEC,eAAe,EAAEC,YAAY,EAAE,GAAGN;IAClE,MAAMH,aAAavB,uBAAuBsB,SACrC3B,+BAAAA,4BACC,CAAC,CAAC,GAAEU,cAAAA,WAAWuB,IAAIhB,KAAK,CAAChB,2BAA2B,aAAhDS,cAAqD,IAAI,aAD9DV,+BAEIM,+BACLS;IACJ,MAAMuB,UAAU5B,WAAWuB,IAAIhB,KAAK,CAACd,wBAAwB;IAC7D,MAAM,EAAEuB,WAAW,EAAE,GAAGS;IAExB,MAAMI,WAAW,CAACC;QAChBN,IAAIO,YAAY,CAAC;QACjBL,gBAAgBI;QAChBN,IAAIQ,SAAS,CAAC,YAAY,GAAGP,OAAOQ,UAAU,GAAG3B,UAAUiB,IAAIhB,KAAK,EAAE,EAAE,GAAG;QAC3EiB,IAAIU,MAAM,CAAC,KAAKC,GAAG;IACrB;IAEA,IAAI,CAACzC,4BAA4B+B,OAAOW,WAAW,GAAG;QACpDP,SAASF,aAAaU,MAAM;QAC5B;IACF;IACA,IAAIT,WAAWZ,eAAeY,YAAYZ,aAAa;QACrDU,gBAAgBC,aAAaU,MAAM;QACnCb,IAAIQ,SAAS,CACX,YACA,GAAGP,OAAOQ,UAAU,GAAG3B,UAAUiB,IAAIhB,KAAK,EAAE;YAC1C;gBAACf;gBAAiBH,mBAAmB4B;aAAM;YAC3C;gBAAC1B;gBAA4B2B;aAAW;YACxC;gBAACzB;gBAAyBuB;aAAY;SACvC,GAAG;QAENQ,IAAIU,MAAM,CAAC,KAAKC,GAAG;QACnB;IACF;IAEA,MAAMG,OAAOvB,iBAAiBC,aAAaC,MAAMC;IACjDQ,gBACEE,WAAWA,YAAYZ,cACnBW,aAAaY,SAAS,GACtBZ,aAAaU,MAAM;IAEzB,IAAIC,MAAM;QACRd,IAAIQ,SAAS,CAAC,QAAQM;QACtB,IAAIf,IAAIiB,OAAO,CAAC,gBAAgB,KAAKF,MAAM;YACzCd,IAAIU,MAAM,CAAC,KAAKC,GAAG;YACnB;QACF;IACF;IAEA,MAAMM,UAAUlB,IAAImB,MAAM,KAAK,QAAQrB,QAAQsB,SAAS,KAAK;IAC7D,MAAM,CAACrB,SAAS,GAAG,MAAMG,OAAOmB,IAAI,CAACC,WAAW,GAAGC,KAAK,CAAC,IAAM;YAAC;SAAK;IACrE,IAAI,CAACxB,UAAU;QACbE,IAAIO,YAAY,CAAC;QACjBL,gBAAgB;QAChBF,IAAIU,MAAM,CAAC,KAAKa,IAAI,CAAC;YAAEC,OAAO;QAAY;QAC1C;IACF;IACA,IAAIC,QAAO3B,iBAAAA,SAAS4B,IAAI,YAAb5B,iBAAiB,KAAKzB,gCAAgC;QAC/DgC,SAASF,aAAaU,MAAM;QAC5B;IACF;IACA,MAAMc,UAAU,MAAMV;IACtB,IAAIU,SAAS;QACX3B,IAAIO,YAAY,CAAC;QACjBP,IAAIQ,SAAS,CAAC,iBAAiB;QAC/BR,IAAIQ,SAAS,CAAC,eAAeoB,OAAOD,QAAQE,iBAAiB;QAC7D7B,IAAIU,MAAM,CAAC,KAAKa,IAAI,CAAC;YAAEC,OAAO;QAAoB;QAClD;IACF;IAEA,IAAIM;IACJ,IAAI;QACF,oEAAoE;QACpE,iEAAiE;QACjE,MAAMC,QAAQ,MAAMxD,kBAAkB0B,OAAOmB,IAAI;QACjDU,OAAO,MAAMxD,eAAe;YAAE2B,QAAQ8B;YAAOtC;YAAMC;QAAW;IAChE,EAAE,OAAO8B,OAAO;QACd,sEAAsE;QACtE,0EAA0E;QAC1EQ,QAAQR,KAAK,CAAC,uCAAuCvB,OAAOQ,UAAU,EAAEe;QACxEnB,SAAS;QACT;IACF;IAEAL,IAAIQ,SAAS,CAAC,gBAAgBsB,KAAKlB,WAAW;IAC9CZ,IAAIQ,SAAS,CAAC,kBAAkBoB,OAAOE,KAAKG,IAAI,CAACC,MAAM;IACvDlC,IAAIQ,SAAS,CACX,uBACA,CAAC,uBAAuB,EAAE3C,mBAAmB4B,MAAM,CAAC,EAClDA,KAAKE,KAAK,KAAK,QAAQ,QAAQ,MAChC,CAAC,CAAC;IAEL,IAAII,IAAImB,MAAM,KAAK,QAAQ;QACzBlB,IAAIU,MAAM,CAAC,KAAKC,GAAG;QACnB;IACF;IACAX,IAAIU,MAAM,CAAC,KAAKC,GAAG,CAACmB,KAAKG,IAAI;AAC/B"}
|
|
@@ -15,6 +15,7 @@
|
|
|
15
15
|
* limitations under the License.
|
|
16
16
|
*/
|
|
17
17
|
import { MEDIA_VARIANT_ENCODER_VERSION_FIELD } from '@aglyn/aglyn/app-utils/media-ref';
|
|
18
|
+
import { type StorageReadableFile } from './storage-read-stream';
|
|
18
19
|
/**
|
|
19
20
|
* Lazy regeneration of an asset's delivery copies (AGL-3486).
|
|
20
21
|
*
|
|
@@ -91,8 +92,7 @@ export interface MediaRegenerationFacts {
|
|
|
91
92
|
export declare function mediaDeliveryCopiesStale(facts: MediaRegenerationFacts, nowMs?: number): boolean;
|
|
92
93
|
/** The slice of a Storage bucket this needs. */
|
|
93
94
|
export interface MediaRegenerationBucket {
|
|
94
|
-
file(path: string): {
|
|
95
|
-
download(): Promise<[Buffer]>;
|
|
95
|
+
file(path: string): StorageReadableFile & {
|
|
96
96
|
save(bytes: Buffer, options: {
|
|
97
97
|
contentType: string;
|
|
98
98
|
metadata: {
|
|
@@ -17,6 +17,7 @@ import { _ as _extends } from "@swc/helpers/_/_extends";
|
|
|
17
17
|
*/ import { FieldValue } from "firebase-admin/firestore";
|
|
18
18
|
import { MEDIA_VARIANT_ENCODER_VERSION, MEDIA_VARIANT_ENCODER_VERSION_FIELD, mediaDisplayObjectPath, mediaVariantEncoderVersionOf } from "@aglyn/aglyn/app-utils/media-ref";
|
|
19
19
|
import { generateMediaVariants, MEDIA_DISPLAY_TYPES, MEDIA_TYPES_WITHOUT_VARIANTS, MEDIA_VARIANT_SOURCE_MAX_BYTES, mediaVariantDocFields, mediaVariantWidthsFor } from "./media-variants.js";
|
|
20
|
+
import { readStorageObject } from "./storage-read-stream.js";
|
|
20
21
|
/**
|
|
21
22
|
* Lazy regeneration of an asset's delivery copies (AGL-3486).
|
|
22
23
|
*
|
|
@@ -147,7 +148,9 @@ import { generateMediaVariants, MEDIA_DISPLAY_TYPES, MEDIA_TYPES_WITHOUT_VARIANT
|
|
|
147
148
|
}).then(()=>undefined);
|
|
148
149
|
let outcome;
|
|
149
150
|
try {
|
|
150
|
-
|
|
151
|
+
// Read through the listener budget, which `file.download()` skips (see
|
|
152
|
+
// `storage-read-stream.ts`).
|
|
153
|
+
const source = await readStorageObject(bucket.file(basePath));
|
|
151
154
|
outcome = await generateMediaVariants({
|
|
152
155
|
buffer: source,
|
|
153
156
|
contentType,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../../../../../../../../libs/tenant/data/admin/src/lib/server/media-delivery-regeneration.ts"],"sourcesContent":["/**\n * @license\n * Copyright 2026 Aglyn LLC\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { FieldValue } from 'firebase-admin/firestore'\nimport {\n MEDIA_VARIANT_ENCODER_VERSION,\n MEDIA_VARIANT_ENCODER_VERSION_FIELD,\n mediaDisplayObjectPath,\n mediaVariantEncoderVersionOf,\n} from '@aglyn/aglyn/app-utils/media-ref'\n\nimport {\n generateMediaVariants,\n MEDIA_DISPLAY_TYPES,\n MEDIA_TYPES_WITHOUT_VARIANTS,\n MEDIA_VARIANT_SOURCE_MAX_BYTES,\n mediaVariantDocFields,\n mediaVariantWidthsFor,\n type SaveMediaDerivedObject,\n} from './media-variants'\n\n/**\n * Lazy regeneration of an asset's delivery copies (AGL-3486).\n *\n * A change to the encoder — its quality, the width ladder, the display copy —\n * has to reach every asset already in every library, and the obvious tool is\n * a backfill script: a pass over the corpus that somebody has to remember to\n * run, against production, after each deploy that changes the encoder. The\n * last one (`backfill-media-variants.mjs`) was written, documented as\n * outstanding and never converged.\n *\n * So the CDN converges the corpus itself. A media document records the\n * encoder generation that made its copies\n * (`MEDIA_VARIANT_ENCODER_VERSION_FIELD`); a request that finds it older than\n * `MEDIA_VARIANT_ENCODER_VERSION` is SERVED what exists — an old variant, or\n * the original — and the asset is regenerated after the response. Every asset\n * a visitor actually sees reaches the new encoder on its first view; one\n * nobody requests costs nothing. An asset that never got variants at all,\n * which is what the old backfill was for, is simply an asset at generation 1.\n *\n * ## Why after the response, and not inside it\n *\n * Regenerating means fetching the original (up to 15 MB) and making up to ten\n * WebP encodes and a display copy: about two seconds for a large photo. A\n * visitor should not wait for that, and nothing about the old bytes is wrong —\n * only heavier — so the request is answered from what exists and the work\n * runs in `after()`.\n *\n * ## One regeneration per asset, not one per visitor\n *\n * A popular image is requested many times before its first regeneration\n * finishes. The first request takes a lease on the document in a transaction\n * (`variantRegeneration.startedAtMs`); every other request in the lease window\n * sees it on the snapshot it already read and schedules nothing, so the lease\n * costs one transaction per asset, not one per request. A failure records the\n * generation it failed at, so a source `sharp` cannot decode is tried once per\n * encoder generation rather than once per minute forever.\n */\n\n/** How long a regeneration's lease keeps others from starting one. */\nexport const MEDIA_REGENERATION_LEASE_MS = 2 * 60 * 1000\n\n/** The document field holding the lease and any failure. */\nexport const MEDIA_REGENERATION_FIELD = 'variantRegeneration'\n\nexport interface MediaRegenerationState {\n /** When the current (or last) regeneration took its lease. */\n startedAtMs?: number\n /** The encoder generation a regeneration last FAILED at. */\n failedVersion?: number\n /** Why, kept short — the same contract as `variantsError`. */\n error?: string\n}\n\n/** The fields of a media document this decision reads, among any others. */\nexport interface MediaRegenerationFacts {\n [field: string]: unknown\n contentType?: unknown\n cdnPath?: unknown\n sizeBytes?: unknown\n width?: unknown\n [MEDIA_VARIANT_ENCODER_VERSION_FIELD]?: unknown\n [MEDIA_REGENERATION_FIELD]?: unknown\n}\n\n/**\n * Whether this asset's delivery copies should be regenerated now.\n *\n * Pure, and decided from the snapshot the CDN already holds, so a request\n * that does not need to regenerate reads nothing more.\n *\n * - Only assets the generator would have worked on at upload: an image with a\n * CDN path (the `mediaCdn` entitlement and a public asset, the same gate the\n * upload routes apply), of a type that has variants or a display copy, no\n * larger than the generator will fetch back.\n * - Only when the recorded generation is older than the current one.\n * - Not while another regeneration holds the lease, and not again at a\n * generation that already failed.\n */\nexport function mediaDeliveryCopiesStale(\n facts: MediaRegenerationFacts,\n nowMs: number = Date.now(),\n): boolean {\n const contentType = typeof facts.contentType === 'string' ? facts.contentType : ''\n if (!contentType.startsWith('image/')) return false\n if (MEDIA_TYPES_WITHOUT_VARIANTS.has(contentType)) return false\n if (typeof facts.cdnPath !== 'string' || !facts.cdnPath) return false\n if (\n mediaVariantEncoderVersionOf(facts[MEDIA_VARIANT_ENCODER_VERSION_FIELD]) >=\n MEDIA_VARIANT_ENCODER_VERSION\n ) {\n return false\n }\n const sizeBytes = Number(facts.sizeBytes ?? 0)\n if (!Number.isFinite(sizeBytes) || sizeBytes > MEDIA_VARIANT_SOURCE_MAX_BYTES) {\n return false\n }\n const sourceWidth =\n typeof facts.width === 'number' && facts.width > 0 ? facts.width : null\n const hasWork =\n mediaVariantWidthsFor({ contentType, sourceWidth }).length > 0 ||\n MEDIA_DISPLAY_TYPES.has(contentType)\n if (!hasWork) return false\n const state = (facts[MEDIA_REGENERATION_FIELD] ?? {}) as MediaRegenerationState\n if (state.failedVersion === MEDIA_VARIANT_ENCODER_VERSION) return false\n if (\n typeof state.startedAtMs === 'number' &&\n nowMs - state.startedAtMs < MEDIA_REGENERATION_LEASE_MS\n ) {\n return false\n }\n return true\n}\n\n/** The slice of a Storage bucket this needs. */\nexport interface MediaRegenerationBucket {\n file(path: string): {\n download(): Promise<[Buffer]>\n save(\n bytes: Buffer,\n options: { contentType: string; metadata: { cacheControl: string } },\n ): Promise<unknown>\n delete(): Promise<unknown>\n }\n}\n\nexport type MediaRegenerationResult =\n | 'regenerated'\n | 'not-stale'\n | 'changed'\n | 'failed'\n\n/**\n * Regenerate one asset's delivery copies with the current encoder.\n *\n * `basePath` is the original's object path as the CDN resolved it — already\n * proved to be inside the asset's own scope (AGL-1881) — and every object this\n * writes or deletes is derived from it.\n *\n * Ordering, and why:\n *\n * 1. **Lease**, in a transaction that re-reads the document: two requests that\n * both saw it stale cannot both start.\n * 2. **Generate and save** the new copies. Their paths are the old ones, so a\n * request in this window may receive new bytes under an old validator —\n * harmless, both are the same picture.\n * 3. **Record**, in a transaction that refuses when the content hash moved: a\n * replace during the work owns the document, and copies of the bytes it\n * replaced must not be recorded over it.\n * 4. **Delete** what the new set no longer names — a width the new encoder\n * skipped, a display copy the new rules do not need — only once the\n * document stops naming them, so the CDN never points at a deleted object.\n */\nexport async function regenerateMediaDeliveryCopies(options: {\n docRef: FirebaseFirestore.DocumentReference\n bucket: MediaRegenerationBucket\n basePath: string\n nowMs?: number\n}): Promise<MediaRegenerationResult> {\n const { docRef, bucket, basePath } = options\n const nowMs = options.nowMs ?? Date.now()\n const { firestore } = docRef\n\n const leased = await firestore.runTransaction(async (transaction) => {\n const snapshot = await transaction.get(docRef)\n const data = snapshot.exists ? (snapshot.data() ?? {}) : null\n if (!data || data['deletedAt'] || !mediaDeliveryCopiesStale(data, nowMs)) {\n return null\n }\n const previous = (data[MEDIA_REGENERATION_FIELD] ?? {}) as MediaRegenerationState\n transaction.update(docRef, {\n [MEDIA_REGENERATION_FIELD]: { ...previous, startedAtMs: nowMs },\n })\n return data\n })\n if (!leased) return 'not-stale'\n\n const contentType = String(leased['contentType'])\n const contentHash = leased['contentHash']\n const previousVariants: number[] = Array.isArray(leased['variants'])\n ? (leased['variants'] as number[])\n : []\n const previousDisplay = Boolean(leased['display'])\n\n const saveVariant: SaveMediaDerivedObject = (path, bytes, type) =>\n bucket\n .file(path)\n .save(bytes, {\n contentType: type,\n metadata: { cacheControl: 'public, max-age=31536000, immutable' },\n })\n .then(() => undefined)\n\n let outcome: Awaited<ReturnType<typeof generateMediaVariants>>\n try {\n const [source] = await bucket.file(basePath).download()\n outcome = await generateMediaVariants({\n buffer: source,\n contentType,\n sourceWidth:\n typeof leased['width'] === 'number' ? (leased['width'] as number) : null,\n objectPath: basePath,\n saveVariant,\n display: true,\n })\n } catch (error) {\n outcome = {\n variants: [],\n error: `source download failed — ${error instanceof Error ? error.message : String(error)}`.slice(0, 300),\n }\n }\n\n if (outcome.error) {\n // The widths that DID land are real objects at the current generation and\n // the ones that did not are still the old encode, so the document names\n // both; the generation stays where it was, and the failure is recorded so\n // it is not retried until the encoder changes again.\n const written = new Set(outcome.variants)\n await firestore.runTransaction(async (transaction) => {\n const fresh = await transaction.get(docRef)\n if (!fresh.exists || fresh.data()?.['contentHash'] !== contentHash) return\n transaction.update(docRef, {\n variants: [...new Set([...previousVariants, ...written])].sort((a, b) => a - b),\n // A display copy that landed before the failure is a real object too.\n ...(outcome.display ? { display: outcome.display } : {}),\n variantsError: outcome.error,\n [MEDIA_REGENERATION_FIELD]: {\n startedAtMs: nowMs,\n failedVersion: MEDIA_VARIANT_ENCODER_VERSION,\n error: outcome.error,\n } satisfies MediaRegenerationState,\n })\n })\n return 'failed'\n }\n\n const recorded = await firestore.runTransaction(async (transaction) => {\n const fresh = await transaction.get(docRef)\n if (!fresh.exists || fresh.data()?.['contentHash'] !== contentHash) return false\n transaction.update(docRef, {\n ...mediaVariantDocFields(outcome),\n variantsError: FieldValue.delete(),\n [MEDIA_REGENERATION_FIELD]: FieldValue.delete(),\n })\n return true\n })\n if (!recorded) return 'changed'\n\n const kept = new Set(outcome.variants)\n await Promise.all([\n ...previousVariants\n .filter((width) => !kept.has(width))\n .map((width) =>\n bucket.file(`${basePath}__w${width}.webp`).delete().catch(() => undefined),\n ),\n ...(previousDisplay && !outcome.display\n ? [bucket.file(mediaDisplayObjectPath(basePath)).delete().catch(() => undefined)]\n : []),\n ])\n return 'regenerated'\n}\n"],"names":["FieldValue","MEDIA_VARIANT_ENCODER_VERSION","MEDIA_VARIANT_ENCODER_VERSION_FIELD","mediaDisplayObjectPath","mediaVariantEncoderVersionOf","generateMediaVariants","MEDIA_DISPLAY_TYPES","MEDIA_TYPES_WITHOUT_VARIANTS","MEDIA_VARIANT_SOURCE_MAX_BYTES","mediaVariantDocFields","mediaVariantWidthsFor","MEDIA_REGENERATION_LEASE_MS","MEDIA_REGENERATION_FIELD","mediaDeliveryCopiesStale","facts","nowMs","Date","now","contentType","startsWith","has","cdnPath","sizeBytes","Number","isFinite","sourceWidth","width","hasWork","length","state","failedVersion","startedAtMs","regenerateMediaDeliveryCopies","options","docRef","bucket","basePath","firestore","leased","runTransaction","transaction","snapshot","data","get","exists","previous","update","String","contentHash","previousVariants","Array","isArray","previousDisplay","Boolean","saveVariant","path","bytes","type","file","save","metadata","cacheControl","then","undefined","outcome","source","download","buffer","objectPath","display","error","variants","Error","message","slice","written","Set","fresh","sort","a","b","variantsError","recorded","delete","kept","Promise","all","filter","map","catch"],"mappings":";AAAA;;;;;;;;;;;;;;;CAeC,GAED,SAASA,UAAU,QAAQ,2BAA0B;AACrD,SACEC,6BAA6B,EAC7BC,mCAAmC,EACnCC,sBAAsB,EACtBC,4BAA4B,QACvB,mCAAkC;AAEzC,SACEC,qBAAqB,EACrBC,mBAAmB,EACnBC,4BAA4B,EAC5BC,8BAA8B,EAC9BC,qBAAqB,EACrBC,qBAAqB,QAEhB,sBAAkB;AAEzB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAoCC,GAED,oEAAoE,GACpE,OAAO,MAAMC,8BAA8B,IAAI,KAAK,KAAI;AAExD,0DAA0D,GAC1D,OAAO,MAAMC,2BAA2B,sBAAqB;AAsB7D;;;;;;;;;;;;;CAaC,GACD,OAAO,SAASC,yBACdC,KAA6B,EAC7BC,QAAgBC,KAAKC,GAAG,EAAE;QAYDH,kBAUVA;IApBf,MAAMI,cAAc,OAAOJ,MAAMI,WAAW,KAAK,WAAWJ,MAAMI,WAAW,GAAG;IAChF,IAAI,CAACA,YAAYC,UAAU,CAAC,WAAW,OAAO;IAC9C,IAAIZ,6BAA6Ba,GAAG,CAACF,cAAc,OAAO;IAC1D,IAAI,OAAOJ,MAAMO,OAAO,KAAK,YAAY,CAACP,MAAMO,OAAO,EAAE,OAAO;IAChE,IACEjB,6BAA6BU,KAAK,CAACZ,oCAAoC,KACvED,+BACA;QACA,OAAO;IACT;IACA,MAAMqB,YAAYC,QAAOT,mBAAAA,MAAMQ,SAAS,YAAfR,mBAAmB;IAC5C,IAAI,CAACS,OAAOC,QAAQ,CAACF,cAAcA,YAAYd,gCAAgC;QAC7E,OAAO;IACT;IACA,MAAMiB,cACJ,OAAOX,MAAMY,KAAK,KAAK,YAAYZ,MAAMY,KAAK,GAAG,IAAIZ,MAAMY,KAAK,GAAG;IACrE,MAAMC,UACJjB,sBAAsB;QAAEQ;QAAaO;IAAY,GAAGG,MAAM,GAAG,KAC7DtB,oBAAoBc,GAAG,CAACF;IAC1B,IAAI,CAACS,SAAS,OAAO;IACrB,MAAME,SAASf,kCAAAA,KAAK,CAACF,yBAAyB,YAA/BE,kCAAmC,CAAC;IACnD,IAAIe,MAAMC,aAAa,KAAK7B,+BAA+B,OAAO;IAClE,IACE,OAAO4B,MAAME,WAAW,KAAK,YAC7BhB,QAAQc,MAAME,WAAW,GAAGpB,6BAC5B;QACA,OAAO;IACT;IACA,OAAO;AACT;AAoBA;;;;;;;;;;;;;;;;;;;;CAoBC,GACD,OAAO,eAAeqB,8BAA8BC,OAKnD;QAEeA;IADd,MAAM,EAAEC,MAAM,EAAEC,MAAM,EAAEC,QAAQ,EAAE,GAAGH;IACrC,MAAMlB,SAAQkB,iBAAAA,QAAQlB,KAAK,YAAbkB,iBAAiBjB,KAAKC,GAAG;IACvC,MAAM,EAAEoB,SAAS,EAAE,GAAGH;IAEtB,MAAMI,SAAS,MAAMD,UAAUE,cAAc,CAAC,OAAOC;YAEnBC,gBAIdC;QALlB,MAAMD,WAAW,MAAMD,YAAYG,GAAG,CAACT;QACvC,MAAMQ,OAAOD,SAASG,MAAM,IAAIH,iBAAAA,SAASC,IAAI,cAAbD,iBAAmB,CAAC,IAAK;QACzD,IAAI,CAACC,QAAQA,IAAI,CAAC,YAAY,IAAI,CAAC7B,yBAAyB6B,MAAM3B,QAAQ;YACxE,OAAO;QACT;QACA,MAAM8B,YAAYH,iCAAAA,IAAI,CAAC9B,yBAAyB,YAA9B8B,iCAAkC,CAAC;QACrDF,YAAYM,MAAM,CAACZ,QAAQ;YACzB,CAACtB,yBAAyB,EAAE,aAAKiC;gBAAUd,aAAahB;;QAC1D;QACA,OAAO2B;IACT;IACA,IAAI,CAACJ,QAAQ,OAAO;IAEpB,MAAMpB,cAAc6B,OAAOT,MAAM,CAAC,cAAc;IAChD,MAAMU,cAAcV,MAAM,CAAC,cAAc;IACzC,MAAMW,mBAA6BC,MAAMC,OAAO,CAACb,MAAM,CAAC,WAAW,IAC9DA,MAAM,CAAC,WAAW,GACnB,EAAE;IACN,MAAMc,kBAAkBC,QAAQf,MAAM,CAAC,UAAU;IAEjD,MAAMgB,cAAsC,CAACC,MAAMC,OAAOC,OACxDtB,OACGuB,IAAI,CAACH,MACLI,IAAI,CAACH,OAAO;YACXtC,aAAauC;YACbG,UAAU;gBAAEC,cAAc;YAAsC;QAClE,GACCC,IAAI,CAAC,IAAMC;IAEhB,IAAIC;IACJ,IAAI;QACF,MAAM,CAACC,OAAO,GAAG,MAAM9B,OAAOuB,IAAI,CAACtB,UAAU8B,QAAQ;QACrDF,UAAU,MAAM3D,sBAAsB;YACpC8D,QAAQF;YACR/C;YACAO,aACE,OAAOa,MAAM,CAAC,QAAQ,KAAK,WAAYA,MAAM,CAAC,QAAQ,GAAc;YACtE8B,YAAYhC;YACZkB;YACAe,SAAS;QACX;IACF,EAAE,OAAOC,OAAO;QACdN,UAAU;YACRO,UAAU,EAAE;YACZD,OAAO,CAAC,yBAAyB,EAAEA,iBAAiBE,QAAQF,MAAMG,OAAO,GAAG1B,OAAOuB,QAAQ,CAACI,KAAK,CAAC,GAAG;QACvG;IACF;IAEA,IAAIV,QAAQM,KAAK,EAAE;QACjB,0EAA0E;QAC1E,wEAAwE;QACxE,0EAA0E;QAC1E,qDAAqD;QACrD,MAAMK,UAAU,IAAIC,IAAIZ,QAAQO,QAAQ;QACxC,MAAMlC,UAAUE,cAAc,CAAC,OAAOC;gBAEfqC;YADrB,MAAMA,QAAQ,MAAMrC,YAAYG,GAAG,CAACT;YACpC,IAAI,CAAC2C,MAAMjC,MAAM,IAAIiC,EAAAA,cAAAA,MAAMnC,IAAI,uBAAVmC,WAAc,CAAC,cAAc,MAAK7B,aAAa;YACpER,YAAYM,MAAM,CAACZ,QAAQ;gBACzBqC,UAAU;uBAAI,IAAIK,IAAI;2BAAI3B;2BAAqB0B;qBAAQ;iBAAE,CAACG,IAAI,CAAC,CAACC,GAAGC,IAAMD,IAAIC;eAEzEhB,QAAQK,OAAO,GAAG;gBAAEA,SAASL,QAAQK,OAAO;YAAC,IAAI,CAAC;gBACtDY,eAAejB,QAAQM,KAAK;gBAC5B,CAAC1D,yBAAyB,EAAE;oBAC1BmB,aAAahB;oBACbe,eAAe7B;oBACfqE,OAAON,QAAQM,KAAK;gBACtB;;QAEJ;QACA,OAAO;IACT;IAEA,MAAMY,WAAW,MAAM7C,UAAUE,cAAc,CAAC,OAAOC;YAEhCqC;QADrB,MAAMA,QAAQ,MAAMrC,YAAYG,GAAG,CAACT;QACpC,IAAI,CAAC2C,MAAMjC,MAAM,IAAIiC,EAAAA,cAAAA,MAAMnC,IAAI,uBAAVmC,WAAc,CAAC,cAAc,MAAK7B,aAAa,OAAO;QAC3ER,YAAYM,MAAM,CAACZ,QAAQ,aACtBzB,sBAAsBuD;YACzBiB,eAAejF,WAAWmF,MAAM;YAChC,CAACvE,yBAAyB,EAAEZ,WAAWmF,MAAM;;QAE/C,OAAO;IACT;IACA,IAAI,CAACD,UAAU,OAAO;IAEtB,MAAME,OAAO,IAAIR,IAAIZ,QAAQO,QAAQ;IACrC,MAAMc,QAAQC,GAAG,CAAC;WACbrC,iBACAsC,MAAM,CAAC,CAAC7D,QAAU,CAAC0D,KAAKhE,GAAG,CAACM,QAC5B8D,GAAG,CAAC,CAAC9D,QACJS,OAAOuB,IAAI,CAAC,GAAGtB,SAAS,GAAG,EAAEV,MAAM,KAAK,CAAC,EAAEyD,MAAM,GAAGM,KAAK,CAAC,IAAM1B;WAEhEX,mBAAmB,CAACY,QAAQK,OAAO,GACnC;YAAClC,OAAOuB,IAAI,CAACvD,uBAAuBiC,WAAW+C,MAAM,GAAGM,KAAK,CAAC,IAAM1B;SAAW,GAC/E,EAAE;KACP;IACD,OAAO;AACT"}
|
|
1
|
+
{"version":3,"sources":["../../../../../../../../libs/tenant/data/admin/src/lib/server/media-delivery-regeneration.ts"],"sourcesContent":["/**\n * @license\n * Copyright 2026 Aglyn LLC\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { FieldValue } from 'firebase-admin/firestore'\nimport {\n MEDIA_VARIANT_ENCODER_VERSION,\n MEDIA_VARIANT_ENCODER_VERSION_FIELD,\n mediaDisplayObjectPath,\n mediaVariantEncoderVersionOf,\n} from '@aglyn/aglyn/app-utils/media-ref'\n\nimport {\n generateMediaVariants,\n MEDIA_DISPLAY_TYPES,\n MEDIA_TYPES_WITHOUT_VARIANTS,\n MEDIA_VARIANT_SOURCE_MAX_BYTES,\n mediaVariantDocFields,\n mediaVariantWidthsFor,\n type SaveMediaDerivedObject,\n} from './media-variants'\nimport { readStorageObject, type StorageReadableFile } from './storage-read-stream'\n\n/**\n * Lazy regeneration of an asset's delivery copies (AGL-3486).\n *\n * A change to the encoder — its quality, the width ladder, the display copy —\n * has to reach every asset already in every library, and the obvious tool is\n * a backfill script: a pass over the corpus that somebody has to remember to\n * run, against production, after each deploy that changes the encoder. The\n * last one (`backfill-media-variants.mjs`) was written, documented as\n * outstanding and never converged.\n *\n * So the CDN converges the corpus itself. A media document records the\n * encoder generation that made its copies\n * (`MEDIA_VARIANT_ENCODER_VERSION_FIELD`); a request that finds it older than\n * `MEDIA_VARIANT_ENCODER_VERSION` is SERVED what exists — an old variant, or\n * the original — and the asset is regenerated after the response. Every asset\n * a visitor actually sees reaches the new encoder on its first view; one\n * nobody requests costs nothing. An asset that never got variants at all,\n * which is what the old backfill was for, is simply an asset at generation 1.\n *\n * ## Why after the response, and not inside it\n *\n * Regenerating means fetching the original (up to 15 MB) and making up to ten\n * WebP encodes and a display copy: about two seconds for a large photo. A\n * visitor should not wait for that, and nothing about the old bytes is wrong —\n * only heavier — so the request is answered from what exists and the work\n * runs in `after()`.\n *\n * ## One regeneration per asset, not one per visitor\n *\n * A popular image is requested many times before its first regeneration\n * finishes. The first request takes a lease on the document in a transaction\n * (`variantRegeneration.startedAtMs`); every other request in the lease window\n * sees it on the snapshot it already read and schedules nothing, so the lease\n * costs one transaction per asset, not one per request. A failure records the\n * generation it failed at, so a source `sharp` cannot decode is tried once per\n * encoder generation rather than once per minute forever.\n */\n\n/** How long a regeneration's lease keeps others from starting one. */\nexport const MEDIA_REGENERATION_LEASE_MS = 2 * 60 * 1000\n\n/** The document field holding the lease and any failure. */\nexport const MEDIA_REGENERATION_FIELD = 'variantRegeneration'\n\nexport interface MediaRegenerationState {\n /** When the current (or last) regeneration took its lease. */\n startedAtMs?: number\n /** The encoder generation a regeneration last FAILED at. */\n failedVersion?: number\n /** Why, kept short — the same contract as `variantsError`. */\n error?: string\n}\n\n/** The fields of a media document this decision reads, among any others. */\nexport interface MediaRegenerationFacts {\n [field: string]: unknown\n contentType?: unknown\n cdnPath?: unknown\n sizeBytes?: unknown\n width?: unknown\n [MEDIA_VARIANT_ENCODER_VERSION_FIELD]?: unknown\n [MEDIA_REGENERATION_FIELD]?: unknown\n}\n\n/**\n * Whether this asset's delivery copies should be regenerated now.\n *\n * Pure, and decided from the snapshot the CDN already holds, so a request\n * that does not need to regenerate reads nothing more.\n *\n * - Only assets the generator would have worked on at upload: an image with a\n * CDN path (the `mediaCdn` entitlement and a public asset, the same gate the\n * upload routes apply), of a type that has variants or a display copy, no\n * larger than the generator will fetch back.\n * - Only when the recorded generation is older than the current one.\n * - Not while another regeneration holds the lease, and not again at a\n * generation that already failed.\n */\nexport function mediaDeliveryCopiesStale(\n facts: MediaRegenerationFacts,\n nowMs: number = Date.now(),\n): boolean {\n const contentType = typeof facts.contentType === 'string' ? facts.contentType : ''\n if (!contentType.startsWith('image/')) return false\n if (MEDIA_TYPES_WITHOUT_VARIANTS.has(contentType)) return false\n if (typeof facts.cdnPath !== 'string' || !facts.cdnPath) return false\n if (\n mediaVariantEncoderVersionOf(facts[MEDIA_VARIANT_ENCODER_VERSION_FIELD]) >=\n MEDIA_VARIANT_ENCODER_VERSION\n ) {\n return false\n }\n const sizeBytes = Number(facts.sizeBytes ?? 0)\n if (!Number.isFinite(sizeBytes) || sizeBytes > MEDIA_VARIANT_SOURCE_MAX_BYTES) {\n return false\n }\n const sourceWidth =\n typeof facts.width === 'number' && facts.width > 0 ? facts.width : null\n const hasWork =\n mediaVariantWidthsFor({ contentType, sourceWidth }).length > 0 ||\n MEDIA_DISPLAY_TYPES.has(contentType)\n if (!hasWork) return false\n const state = (facts[MEDIA_REGENERATION_FIELD] ?? {}) as MediaRegenerationState\n if (state.failedVersion === MEDIA_VARIANT_ENCODER_VERSION) return false\n if (\n typeof state.startedAtMs === 'number' &&\n nowMs - state.startedAtMs < MEDIA_REGENERATION_LEASE_MS\n ) {\n return false\n }\n return true\n}\n\n/** The slice of a Storage bucket this needs. */\nexport interface MediaRegenerationBucket {\n file(path: string): StorageReadableFile & {\n save(\n bytes: Buffer,\n options: { contentType: string; metadata: { cacheControl: string } },\n ): Promise<unknown>\n delete(): Promise<unknown>\n }\n}\n\nexport type MediaRegenerationResult =\n | 'regenerated'\n | 'not-stale'\n | 'changed'\n | 'failed'\n\n/**\n * Regenerate one asset's delivery copies with the current encoder.\n *\n * `basePath` is the original's object path as the CDN resolved it — already\n * proved to be inside the asset's own scope (AGL-1881) — and every object this\n * writes or deletes is derived from it.\n *\n * Ordering, and why:\n *\n * 1. **Lease**, in a transaction that re-reads the document: two requests that\n * both saw it stale cannot both start.\n * 2. **Generate and save** the new copies. Their paths are the old ones, so a\n * request in this window may receive new bytes under an old validator —\n * harmless, both are the same picture.\n * 3. **Record**, in a transaction that refuses when the content hash moved: a\n * replace during the work owns the document, and copies of the bytes it\n * replaced must not be recorded over it.\n * 4. **Delete** what the new set no longer names — a width the new encoder\n * skipped, a display copy the new rules do not need — only once the\n * document stops naming them, so the CDN never points at a deleted object.\n */\nexport async function regenerateMediaDeliveryCopies(options: {\n docRef: FirebaseFirestore.DocumentReference\n bucket: MediaRegenerationBucket\n basePath: string\n nowMs?: number\n}): Promise<MediaRegenerationResult> {\n const { docRef, bucket, basePath } = options\n const nowMs = options.nowMs ?? Date.now()\n const { firestore } = docRef\n\n const leased = await firestore.runTransaction(async (transaction) => {\n const snapshot = await transaction.get(docRef)\n const data = snapshot.exists ? (snapshot.data() ?? {}) : null\n if (!data || data['deletedAt'] || !mediaDeliveryCopiesStale(data, nowMs)) {\n return null\n }\n const previous = (data[MEDIA_REGENERATION_FIELD] ?? {}) as MediaRegenerationState\n transaction.update(docRef, {\n [MEDIA_REGENERATION_FIELD]: { ...previous, startedAtMs: nowMs },\n })\n return data\n })\n if (!leased) return 'not-stale'\n\n const contentType = String(leased['contentType'])\n const contentHash = leased['contentHash']\n const previousVariants: number[] = Array.isArray(leased['variants'])\n ? (leased['variants'] as number[])\n : []\n const previousDisplay = Boolean(leased['display'])\n\n const saveVariant: SaveMediaDerivedObject = (path, bytes, type) =>\n bucket\n .file(path)\n .save(bytes, {\n contentType: type,\n metadata: { cacheControl: 'public, max-age=31536000, immutable' },\n })\n .then(() => undefined)\n\n let outcome: Awaited<ReturnType<typeof generateMediaVariants>>\n try {\n // Read through the listener budget, which `file.download()` skips (see\n // `storage-read-stream.ts`).\n const source = await readStorageObject(bucket.file(basePath))\n outcome = await generateMediaVariants({\n buffer: source,\n contentType,\n sourceWidth:\n typeof leased['width'] === 'number' ? (leased['width'] as number) : null,\n objectPath: basePath,\n saveVariant,\n display: true,\n })\n } catch (error) {\n outcome = {\n variants: [],\n error: `source download failed — ${error instanceof Error ? error.message : String(error)}`.slice(0, 300),\n }\n }\n\n if (outcome.error) {\n // The widths that DID land are real objects at the current generation and\n // the ones that did not are still the old encode, so the document names\n // both; the generation stays where it was, and the failure is recorded so\n // it is not retried until the encoder changes again.\n const written = new Set(outcome.variants)\n await firestore.runTransaction(async (transaction) => {\n const fresh = await transaction.get(docRef)\n if (!fresh.exists || fresh.data()?.['contentHash'] !== contentHash) return\n transaction.update(docRef, {\n variants: [...new Set([...previousVariants, ...written])].sort((a, b) => a - b),\n // A display copy that landed before the failure is a real object too.\n ...(outcome.display ? { display: outcome.display } : {}),\n variantsError: outcome.error,\n [MEDIA_REGENERATION_FIELD]: {\n startedAtMs: nowMs,\n failedVersion: MEDIA_VARIANT_ENCODER_VERSION,\n error: outcome.error,\n } satisfies MediaRegenerationState,\n })\n })\n return 'failed'\n }\n\n const recorded = await firestore.runTransaction(async (transaction) => {\n const fresh = await transaction.get(docRef)\n if (!fresh.exists || fresh.data()?.['contentHash'] !== contentHash) return false\n transaction.update(docRef, {\n ...mediaVariantDocFields(outcome),\n variantsError: FieldValue.delete(),\n [MEDIA_REGENERATION_FIELD]: FieldValue.delete(),\n })\n return true\n })\n if (!recorded) return 'changed'\n\n const kept = new Set(outcome.variants)\n await Promise.all([\n ...previousVariants\n .filter((width) => !kept.has(width))\n .map((width) =>\n bucket.file(`${basePath}__w${width}.webp`).delete().catch(() => undefined),\n ),\n ...(previousDisplay && !outcome.display\n ? [bucket.file(mediaDisplayObjectPath(basePath)).delete().catch(() => undefined)]\n : []),\n ])\n return 'regenerated'\n}\n"],"names":["FieldValue","MEDIA_VARIANT_ENCODER_VERSION","MEDIA_VARIANT_ENCODER_VERSION_FIELD","mediaDisplayObjectPath","mediaVariantEncoderVersionOf","generateMediaVariants","MEDIA_DISPLAY_TYPES","MEDIA_TYPES_WITHOUT_VARIANTS","MEDIA_VARIANT_SOURCE_MAX_BYTES","mediaVariantDocFields","mediaVariantWidthsFor","readStorageObject","MEDIA_REGENERATION_LEASE_MS","MEDIA_REGENERATION_FIELD","mediaDeliveryCopiesStale","facts","nowMs","Date","now","contentType","startsWith","has","cdnPath","sizeBytes","Number","isFinite","sourceWidth","width","hasWork","length","state","failedVersion","startedAtMs","regenerateMediaDeliveryCopies","options","docRef","bucket","basePath","firestore","leased","runTransaction","transaction","snapshot","data","get","exists","previous","update","String","contentHash","previousVariants","Array","isArray","previousDisplay","Boolean","saveVariant","path","bytes","type","file","save","metadata","cacheControl","then","undefined","outcome","source","buffer","objectPath","display","error","variants","Error","message","slice","written","Set","fresh","sort","a","b","variantsError","recorded","delete","kept","Promise","all","filter","map","catch"],"mappings":";AAAA;;;;;;;;;;;;;;;CAeC,GAED,SAASA,UAAU,QAAQ,2BAA0B;AACrD,SACEC,6BAA6B,EAC7BC,mCAAmC,EACnCC,sBAAsB,EACtBC,4BAA4B,QACvB,mCAAkC;AAEzC,SACEC,qBAAqB,EACrBC,mBAAmB,EACnBC,4BAA4B,EAC5BC,8BAA8B,EAC9BC,qBAAqB,EACrBC,qBAAqB,QAEhB,sBAAkB;AACzB,SAASC,iBAAiB,QAAkC,2BAAuB;AAEnF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAoCC,GAED,oEAAoE,GACpE,OAAO,MAAMC,8BAA8B,IAAI,KAAK,KAAI;AAExD,0DAA0D,GAC1D,OAAO,MAAMC,2BAA2B,sBAAqB;AAsB7D;;;;;;;;;;;;;CAaC,GACD,OAAO,SAASC,yBACdC,KAA6B,EAC7BC,QAAgBC,KAAKC,GAAG,EAAE;QAYDH,kBAUVA;IApBf,MAAMI,cAAc,OAAOJ,MAAMI,WAAW,KAAK,WAAWJ,MAAMI,WAAW,GAAG;IAChF,IAAI,CAACA,YAAYC,UAAU,CAAC,WAAW,OAAO;IAC9C,IAAIb,6BAA6Bc,GAAG,CAACF,cAAc,OAAO;IAC1D,IAAI,OAAOJ,MAAMO,OAAO,KAAK,YAAY,CAACP,MAAMO,OAAO,EAAE,OAAO;IAChE,IACElB,6BAA6BW,KAAK,CAACb,oCAAoC,KACvED,+BACA;QACA,OAAO;IACT;IACA,MAAMsB,YAAYC,QAAOT,mBAAAA,MAAMQ,SAAS,YAAfR,mBAAmB;IAC5C,IAAI,CAACS,OAAOC,QAAQ,CAACF,cAAcA,YAAYf,gCAAgC;QAC7E,OAAO;IACT;IACA,MAAMkB,cACJ,OAAOX,MAAMY,KAAK,KAAK,YAAYZ,MAAMY,KAAK,GAAG,IAAIZ,MAAMY,KAAK,GAAG;IACrE,MAAMC,UACJlB,sBAAsB;QAAES;QAAaO;IAAY,GAAGG,MAAM,GAAG,KAC7DvB,oBAAoBe,GAAG,CAACF;IAC1B,IAAI,CAACS,SAAS,OAAO;IACrB,MAAME,SAASf,kCAAAA,KAAK,CAACF,yBAAyB,YAA/BE,kCAAmC,CAAC;IACnD,IAAIe,MAAMC,aAAa,KAAK9B,+BAA+B,OAAO;IAClE,IACE,OAAO6B,MAAME,WAAW,KAAK,YAC7BhB,QAAQc,MAAME,WAAW,GAAGpB,6BAC5B;QACA,OAAO;IACT;IACA,OAAO;AACT;AAmBA;;;;;;;;;;;;;;;;;;;;CAoBC,GACD,OAAO,eAAeqB,8BAA8BC,OAKnD;QAEeA;IADd,MAAM,EAAEC,MAAM,EAAEC,MAAM,EAAEC,QAAQ,EAAE,GAAGH;IACrC,MAAMlB,SAAQkB,iBAAAA,QAAQlB,KAAK,YAAbkB,iBAAiBjB,KAAKC,GAAG;IACvC,MAAM,EAAEoB,SAAS,EAAE,GAAGH;IAEtB,MAAMI,SAAS,MAAMD,UAAUE,cAAc,CAAC,OAAOC;YAEnBC,gBAIdC;QALlB,MAAMD,WAAW,MAAMD,YAAYG,GAAG,CAACT;QACvC,MAAMQ,OAAOD,SAASG,MAAM,IAAIH,iBAAAA,SAASC,IAAI,cAAbD,iBAAmB,CAAC,IAAK;QACzD,IAAI,CAACC,QAAQA,IAAI,CAAC,YAAY,IAAI,CAAC7B,yBAAyB6B,MAAM3B,QAAQ;YACxE,OAAO;QACT;QACA,MAAM8B,YAAYH,iCAAAA,IAAI,CAAC9B,yBAAyB,YAA9B8B,iCAAkC,CAAC;QACrDF,YAAYM,MAAM,CAACZ,QAAQ;YACzB,CAACtB,yBAAyB,EAAE,aAAKiC;gBAAUd,aAAahB;;QAC1D;QACA,OAAO2B;IACT;IACA,IAAI,CAACJ,QAAQ,OAAO;IAEpB,MAAMpB,cAAc6B,OAAOT,MAAM,CAAC,cAAc;IAChD,MAAMU,cAAcV,MAAM,CAAC,cAAc;IACzC,MAAMW,mBAA6BC,MAAMC,OAAO,CAACb,MAAM,CAAC,WAAW,IAC9DA,MAAM,CAAC,WAAW,GACnB,EAAE;IACN,MAAMc,kBAAkBC,QAAQf,MAAM,CAAC,UAAU;IAEjD,MAAMgB,cAAsC,CAACC,MAAMC,OAAOC,OACxDtB,OACGuB,IAAI,CAACH,MACLI,IAAI,CAACH,OAAO;YACXtC,aAAauC;YACbG,UAAU;gBAAEC,cAAc;YAAsC;QAClE,GACCC,IAAI,CAAC,IAAMC;IAEhB,IAAIC;IACJ,IAAI;QACF,uEAAuE;QACvE,6BAA6B;QAC7B,MAAMC,SAAS,MAAMvD,kBAAkByB,OAAOuB,IAAI,CAACtB;QACnD4B,UAAU,MAAM5D,sBAAsB;YACpC8D,QAAQD;YACR/C;YACAO,aACE,OAAOa,MAAM,CAAC,QAAQ,KAAK,WAAYA,MAAM,CAAC,QAAQ,GAAc;YACtE6B,YAAY/B;YACZkB;YACAc,SAAS;QACX;IACF,EAAE,OAAOC,OAAO;QACdL,UAAU;YACRM,UAAU,EAAE;YACZD,OAAO,CAAC,yBAAyB,EAAEA,iBAAiBE,QAAQF,MAAMG,OAAO,GAAGzB,OAAOsB,QAAQ,CAACI,KAAK,CAAC,GAAG;QACvG;IACF;IAEA,IAAIT,QAAQK,KAAK,EAAE;QACjB,0EAA0E;QAC1E,wEAAwE;QACxE,0EAA0E;QAC1E,qDAAqD;QACrD,MAAMK,UAAU,IAAIC,IAAIX,QAAQM,QAAQ;QACxC,MAAMjC,UAAUE,cAAc,CAAC,OAAOC;gBAEfoC;YADrB,MAAMA,QAAQ,MAAMpC,YAAYG,GAAG,CAACT;YACpC,IAAI,CAAC0C,MAAMhC,MAAM,IAAIgC,EAAAA,cAAAA,MAAMlC,IAAI,uBAAVkC,WAAc,CAAC,cAAc,MAAK5B,aAAa;YACpER,YAAYM,MAAM,CAACZ,QAAQ;gBACzBoC,UAAU;uBAAI,IAAIK,IAAI;2BAAI1B;2BAAqByB;qBAAQ;iBAAE,CAACG,IAAI,CAAC,CAACC,GAAGC,IAAMD,IAAIC;eAEzEf,QAAQI,OAAO,GAAG;gBAAEA,SAASJ,QAAQI,OAAO;YAAC,IAAI,CAAC;gBACtDY,eAAehB,QAAQK,KAAK;gBAC5B,CAACzD,yBAAyB,EAAE;oBAC1BmB,aAAahB;oBACbe,eAAe9B;oBACfqE,OAAOL,QAAQK,KAAK;gBACtB;;QAEJ;QACA,OAAO;IACT;IAEA,MAAMY,WAAW,MAAM5C,UAAUE,cAAc,CAAC,OAAOC;YAEhCoC;QADrB,MAAMA,QAAQ,MAAMpC,YAAYG,GAAG,CAACT;QACpC,IAAI,CAAC0C,MAAMhC,MAAM,IAAIgC,EAAAA,cAAAA,MAAMlC,IAAI,uBAAVkC,WAAc,CAAC,cAAc,MAAK5B,aAAa,OAAO;QAC3ER,YAAYM,MAAM,CAACZ,QAAQ,aACtB1B,sBAAsBwD;YACzBgB,eAAejF,WAAWmF,MAAM;YAChC,CAACtE,yBAAyB,EAAEb,WAAWmF,MAAM;;QAE/C,OAAO;IACT;IACA,IAAI,CAACD,UAAU,OAAO;IAEtB,MAAME,OAAO,IAAIR,IAAIX,QAAQM,QAAQ;IACrC,MAAMc,QAAQC,GAAG,CAAC;WACbpC,iBACAqC,MAAM,CAAC,CAAC5D,QAAU,CAACyD,KAAK/D,GAAG,CAACM,QAC5B6D,GAAG,CAAC,CAAC7D,QACJS,OAAOuB,IAAI,CAAC,GAAGtB,SAAS,GAAG,EAAEV,MAAM,KAAK,CAAC,EAAEwD,MAAM,GAAGM,KAAK,CAAC,IAAMzB;WAEhEX,mBAAmB,CAACY,QAAQI,OAAO,GACnC;YAACjC,OAAOuB,IAAI,CAACxD,uBAAuBkC,WAAW8C,MAAM,GAAGM,KAAK,CAAC,IAAMzB;SAAW,GAC/E,EAAE;KACP;IACD,OAAO;AACT"}
|
|
@@ -238,6 +238,33 @@ const firestoreApp = ()=>firebaseAdmin.app().firestore();
|
|
|
238
238
|
exported: true,
|
|
239
239
|
note: 'Commerce events still owed to a connected marketing platform (AGL-3639): checkout started, order paid, fulfilled, refunded or cancelled, with the shopper’s email, the order, its items and value, and the tracking number. A SHOPPER’s data held for the merchant as controller, so disclosed to the ORG only, for the reason `supplierDeliveries` gives; a row is deleted once delivered.'
|
|
240
240
|
},
|
|
241
|
+
{
|
|
242
|
+
collection: 'helpDeskConnections',
|
|
243
|
+
keyedBy: 'field',
|
|
244
|
+
subjects: [
|
|
245
|
+
'org'
|
|
246
|
+
],
|
|
247
|
+
exported: true,
|
|
248
|
+
note: 'EXISTENCE ONLY for the credential — each site’s connection to the merchant’s own Gorgias, Re:amaze, Kustomer, Gladly or Richpanel account (AGL-3824): provider, status, the account’s subdomain and name, the Richpanel support address, the chosen channel, inbox or custom object, the sync switches, cursors and run timing. The sealed API key, login and app client id, sealed together, and the id of the key that sealed them carry `token` in their names and are redacted (see redactSecrets); the document id is the site id and the provider.'
|
|
249
|
+
},
|
|
250
|
+
{
|
|
251
|
+
collection: 'helpDeskOutbox',
|
|
252
|
+
keyedBy: 'field',
|
|
253
|
+
subjects: [
|
|
254
|
+
'org'
|
|
255
|
+
],
|
|
256
|
+
exported: true,
|
|
257
|
+
note: 'Orders and contact form messages still owed to a connected help desk (AGL-3824): a buyer’s email and name with the order’s number, items and totals, or a message’s sender, subject and text. A SHOPPER’s or a visitor’s data held for the merchant as controller, so disclosed to the ORG only, for the reason `supplierDeliveries` gives; a row is deleted once delivered.'
|
|
258
|
+
},
|
|
259
|
+
{
|
|
260
|
+
collection: 'helpDeskCustomers',
|
|
261
|
+
keyedBy: 'field',
|
|
262
|
+
subjects: [
|
|
263
|
+
'org'
|
|
264
|
+
],
|
|
265
|
+
exported: true,
|
|
266
|
+
note: 'What each help desk connection knows of a buyer (AGL-3824): their email and name, the help desk’s id for them and their ten latest orders, so each order sent carries the recent ones. A SHOPPER’s data held for the merchant as controller, so disclosed to the ORG only, for the reason `supplierDeliveries` gives. The document id carries a truncated SHA-256 of the email — a lookup key, not a credential.'
|
|
267
|
+
},
|
|
241
268
|
{
|
|
242
269
|
collection: 'adConversionConnections',
|
|
243
270
|
keyedBy: 'field',
|
|
@@ -1082,8 +1109,8 @@ function sourcesFor(subject) {
|
|
|
1082
1109
|
data['outreachMailboxCredentials'] = await readByField(db, 'outreachMailboxCredentials', 'orgId', orgId);
|
|
1083
1110
|
// Where each short tracking link the org's sequences sent points (AGL-3297).
|
|
1084
1111
|
data['outreachLinks'] = await readByField(db, 'outreachLinks', 'orgId', orgId);
|
|
1085
|
-
// Shipping (AGL-3612), tax services (AGL-3631)
|
|
1086
|
-
// (AGL-3639): top-level and keyed by `orgId`, the bound their
|
|
1112
|
+
// Shipping (AGL-3612), tax services (AGL-3631), marketing platforms
|
|
1113
|
+
// (AGL-3639) and help desks (AGL-3824): top-level and keyed by `orgId`, the bound their
|
|
1087
1114
|
// `orgKeyedCollections` erasure sweeps by. The sealed credentials are
|
|
1088
1115
|
// withheld by `redactSecrets` on their names.
|
|
1089
1116
|
for (const collection of [
|
|
@@ -1095,6 +1122,9 @@ function sourcesFor(subject) {
|
|
|
1095
1122
|
'taxEngineTransactions',
|
|
1096
1123
|
'marketingPlatformConnections',
|
|
1097
1124
|
'marketingPlatformEvents',
|
|
1125
|
+
'helpDeskConnections',
|
|
1126
|
+
'helpDeskOutbox',
|
|
1127
|
+
'helpDeskCustomers',
|
|
1098
1128
|
'adConversionConnections',
|
|
1099
1129
|
'adConversionConsents',
|
|
1100
1130
|
'adConversionEvents',
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../../../../../../../../libs/tenant/data/admin/src/lib/server/personal-data-export.ts"],"sourcesContent":["/**\n * @license\n * Copyright 2026 Aglyn LLC\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n/**\n * Personal-data export — access and portability, in a machine-readable form\n * (AGL-1974).\n *\n * Privacy Policy §7 grants portability (GDPR Art. 20) and nothing in the\n * product exported a person's or a workspace's personal data. Six surfaces\n * told customers to \"export first\" before deleting — the org settings Delete\n * tab, the Close account card, `erase-org-cli.mjs`, the staff org actions,\n * live DPA §11 and Terms §13.3 — all pointing at something that did not\n * exist. An access request was assembled by a staff member reading the console\n * by hand.\n *\n * ## THE PROPERTY THIS FILE EXISTS FOR\n *\n * **An export whose coverage differs from erasure's is a bug in one of them.**\n * Both answer the same question — *what do we hold about this subject* — and\n * the honest failure modes are symmetric: an export that misses a collection\n * under-discloses on an access request, and an erasure that misses the same\n * one leaves data behind after we said it was gone. Historically the second\n * kept happening (AGL-1444, AGL-1448, AGL-1970, AGL-1971 each found a\n * collection outside the cascade), which is exactly the evidence that a list\n * kept in one place and mirrored by hand in another will drift.\n *\n * The ideal is one enumeration both derive from. That is not available here:\n * `erase.ts` is under concurrent change and its sweep list is expressed as\n * code, not data. So the coverage is enumerated ONCE below, as data, and\n * `personal-data-export-coverage.spec.ts` DISCOVERS every collection\n * `erase.ts` touches out of its source and fails when the two disagree. A new\n * sweep added to the erasure turns this file red until somebody decides\n * whether the export discloses it — which is the decision that was being\n * skipped.\n *\n * That guard is worth more than the feature.\n *\n * ## What must NOT come out wholesale\n *\n * Three categories, and each is a way an access request becomes a breach:\n *\n * 1. **Another person's data.** A workspace is shared. A member asking what\n * we hold about THEM gets their own roster row, not the roster; their own\n * support messages, not the thread's other authors. Handled by exporting\n * the person and the workspace as different subjects with different\n * scopes, never by filtering a single blob after the fact.\n * 2. **Secrets.** An `apiKeys` row confirms that a credential exists — its\n * label, scopes, creator and dates, which are all facts about the subject\n * — and never re-discloses the credential. Note the document ID *is* the\n * SHA-256 of the token, so the id itself is withheld too; a hash handed\n * back is a verifier handed back.\n * 3. **Staff-only fields.** Internal notes, override reasons and moderation\n * state are about our handling, not about the person, and several name a\n * staff member.\n *\n * ## What it is NOT\n *\n * Not `GET /api/hosts/export` — that is a Pro+ site-design backup whose own\n * header says it never includes bookings, leads, submissions or secrets, and\n * offering it as a portability answer would be misdescribing it. Not the\n * `firestore-export` cron, which is an unscoped DR snapshot. Not the audit CSV.\n */\n\nimport { firebaseAdmin } from './firebase-admin'\nimport { CONSOLE_DOMAINS_COLLECTION } from './console-domains'\n\nconst firestoreApp = () => firebaseAdmin.app().firestore()\n\n/** Which subject an export source belongs to. */\nexport type ExportSubject = 'user' | 'org'\n\n/** How a source is located — the axis a path cascade is blind to. */\nexport type ExportKeying =\n /** Top-level collection carrying the subject id in a FIELD. */\n | 'field'\n /** Top-level collection whose DOCUMENT ID is the subject id. */\n | 'doc-id'\n /** Under the subject's own document path. */\n | 'path'\n /** Reached through another collection's rows (a collection group). */\n | 'collection-group'\n /** Outside Firestore entirely. */\n | 'external'\n\nexport interface ExportSourceSpec {\n /** Firestore collection name, or an `external:` pseudo-name. */\n collection: string\n keyedBy: ExportKeying\n subjects: readonly ExportSubject[]\n /**\n * `true` when the export discloses it. `false` requires `reason` — and the\n * reason is the whole value of the field: it is a decision recorded at the\n * moment it was made, not an omission discovered later.\n */\n exported: boolean\n /** Why it is withheld, or how it is disclosed. Always present. */\n note: string\n}\n\n/**\n * Every collection `erase.ts` touches, and what the export does with it.\n *\n * Kept in the SAME order as the erasure's steps so a reader comparing the two\n * files reads them in step. `personal-data-export-coverage.spec.ts` proves the\n * set matches; this list carries the decision for each one.\n */\nexport const PERSONAL_DATA_SOURCES: readonly ExportSourceSpec[] = [\n // ── The person ────────────────────────────────────────────────────────\n {\n collection: 'users',\n keyedBy: 'doc-id',\n subjects: ['user'],\n exported: true,\n note: 'Profile: name, phone, postal address, legalAcceptances, and every subcollection (orgs, hostMemberships, notifications, passkeys).',\n },\n {\n collection: 'profiles',\n keyedBy: 'doc-id',\n subjects: ['user'],\n exported: true,\n note: 'The world-readable handle, display name and Stripe Connect account. Not reachable from the user tree (AGL-1970) — the same blind spot on both sides.',\n },\n // ── The workspace ─────────────────────────────────────────────────────\n {\n collection: 'orgs',\n keyedBy: 'doc-id',\n subjects: ['org'],\n exported: true,\n note: 'The org document and every subcollection: members, invites, roles, usage, apiUsage, analytics, datasets/records, contacts, lists, media, installs, activity, assistExchanges, retention.',\n },\n {\n collection: 'hosts',\n keyedBy: 'field',\n subjects: ['org'],\n exported: true,\n note: 'Each site tree the org owns — pages, layouts, orders, form submissions. Secrets within are redacted, not omitted, so the export still says a webhook exists.',\n },\n {\n collection: 'hostIndex',\n keyedBy: 'doc-id',\n subjects: ['org'],\n exported: true,\n note: 'Subdomain/CNAME routing for each of the org’s sites. Configuration the customer set, so it is theirs to take.',\n },\n {\n collection: 'supplierDeliveries',\n keyedBy: 'field',\n subjects: ['org'],\n exported: true,\n note: 'The dropship outbox for each of the org’s sites, keyed by hostId (AGL-1448). Disclosed to the ORG and to no one else, and the reason is the unusual one: the personal data in the row is a SHOPPER’s — name, email and shipping address in the body we sign and POST to the supplier. That is the merchant’s own order record, which they already read in their console and for which they are the controller and we the processor, so an org access request is entitled to it. It is NOT a `user`-subject source: a console member asking what we hold about THEM is not the shopper, and returning these rows would answer their request with a third party’s address. Withholding it was not an option worth taking — `eraseHost` destroys these rows, so an export that omitted them would delete what it never disclosed.',\n },\n // ── Top-level, field-keyed: what a path cascade cannot see ────────────\n {\n collection: 'apiKeys',\n keyedBy: 'field',\n subjects: ['org'],\n exported: true,\n note: 'EXISTENCE ONLY — label, scopes, creating uid, dates. Never the credential, and never the document id, which is the SHA-256 of the token.',\n },\n {\n collection: 'outreachMailboxCredentials',\n keyedBy: 'field',\n subjects: ['org'],\n exported: true,\n note: 'EXISTENCE ONLY — which connected mailbox a credential authorizes, its provider, granted scopes and dates (AGL-2974), and since AGL-2978 the connected account’s address, its Google account id and the member who connected it: the organization’s own record of whose mailbox it holds. Every token is redacted by name and by shape (see redactSecrets) and never disclosed; the sealed refresh token and the id of the key that sealed it both carry `token` in their names. The document id is the mailbox id, not derived from any secret, so it is disclosed.',\n },\n {\n collection: 'outreachLinks',\n keyedBy: 'field',\n subjects: ['org'],\n exported: true,\n note: 'Where each short tracking link in a sent Outreach sequence email points (AGL-3297): the enrollment, the step, the link’s index and the destination the step wrote. The organization’s own record of what it sent; the id is random and derived from no secret.',\n },\n {\n collection: 'shippingTrackers',\n keyedBy: 'field',\n subjects: ['org'],\n exported: true,\n note: 'One row per tracking number a label was bought for (AGL-3612): the site, the order record, the carrier, the tracking number and its latest status. The organization’s own record of what it shipped; the order it points at is already disclosed under its site. Org-only, for the reason `supplierDeliveries` gives: a tracking number locates a shopper’s parcel.',\n },\n {\n collection: 'shippingQuoteCache',\n keyedBy: 'field',\n subjects: ['org'],\n exported: true,\n note: 'Carrier quotes for a site’s checkout, kept ten minutes (AGL-3612): the site, each service’s price and transit estimate, and when the row expires. No address is stored; the document id is a SHA-256 of the quoted request, an identifier rather than a verifier of any credential. Disclosed while it exists so the export does not omit a collection the erasure destroys.',\n },\n {\n collection: 'taxEngineConnections',\n keyedBy: 'field',\n subjects: ['org'],\n exported: true,\n note: 'EXISTENCE ONLY for the credential — each site’s connection to the merchant’s own Avalara AvaTax or TaxJar account (AGL-3631): provider, environment, account id and company code, ship-from address, default tax code, last test result and the member who connected it. The sealed API token and the id of the key that sealed it both carry `token` in their names and are redacted (see redactSecrets); the document id is the site id.',\n },\n {\n collection: 'taxEngineProductCodes',\n keyedBy: 'field',\n subjects: ['org'],\n exported: true,\n note: 'The tax code the merchant assigned each product at their tax service (AGL-3631). Catalog configuration the customer set, so it is theirs to take.',\n },\n {\n collection: 'taxEngineExemptions',\n keyedBy: 'field',\n subjects: ['org'],\n exported: true,\n note: 'Customers the merchant recorded as tax-exempt (AGL-3631): email, name, exemption type, certificate number and the regions it covers. A SHOPPER’s data held for the merchant as controller, so disclosed to the ORG only, for the reason `supplierDeliveries` gives. The document id carries a truncated SHA-256 of the email — a lookup key, not a credential.',\n },\n {\n collection: 'taxEngineTransactions',\n keyedBy: 'field',\n subjects: ['org'],\n exported: true,\n note: 'Each paid order as recorded with the merchant’s tax service, and the refunds reversed against it (AGL-3631): the sale sent (lines, ship-to address, the customer code, which is the shopper’s email), its status and last error. The merchant’s own tax record and a shopper’s address, so disclosed to the ORG only.',\n },\n {\n collection: 'marketingPlatformConnections',\n keyedBy: 'field',\n subjects: ['org'],\n exported: true,\n note: 'EXISTENCE ONLY for the credential — each site’s connection to the merchant’s own Mailchimp, Klaviyo, Omnisend or Attentive account (AGL-3639): provider, status, account name, the chosen list and tag, the sync switches, cursors and run timing. The sealed access and refresh tokens and the id of the key that sealed them carry `token` in their names and are redacted (see redactSecrets); the document id is the site id and the provider.',\n },\n {\n collection: 'marketingPlatformEvents',\n keyedBy: 'field',\n subjects: ['org'],\n exported: true,\n note: 'Commerce events still owed to a connected marketing platform (AGL-3639): checkout started, order paid, fulfilled, refunded or cancelled, with the shopper’s email, the order, its items and value, and the tracking number. A SHOPPER’s data held for the merchant as controller, so disclosed to the ORG only, for the reason `supplierDeliveries` gives; a row is deleted once delivered.',\n },\n {\n collection: 'adConversionConnections',\n keyedBy: 'field',\n subjects: ['org'],\n exported: true,\n note: 'EXISTENCE ONLY for the credential — each site’s Conversions API connection to the merchant’s own Meta, TikTok or Pinterest ad account (AGL-3694): provider, status, the Pinterest ad account id, the test event code, and what was last sent or refused. The sealed access token and the id of the key that sealed it carry `token` in their names and are redacted (see redactSecrets); the document id is the site id and the provider.',\n },\n {\n collection: 'adConversionConsents',\n keyedBy: 'field',\n subjects: ['org'],\n exported: true,\n note: 'The advertising consent a shopper’s checkout carried, recorded only where it granted advertising on a site with a connection (AGL-3694): the consent status and when it was decided, the country, the page, the ad vendors’ browser ids, the IP address and the user agent. A SHOPPER’s data held for the merchant as controller, so disclosed to the ORG only; deleted when the order is reported or after 14 days.',\n },\n {\n collection: 'adConversionEvents',\n keyedBy: 'field',\n subjects: ['org'],\n exported: true,\n note: 'Purchases and leads owed to a connected ad account (AGL-3694): the event id, order id, value and items, the visitor’s details as SHA-256 hashes, and the IP address, user agent and vendor browser ids. A SHOPPER’s data held for the merchant as controller, so disclosed to the ORG only; the personal fields are dropped once the event is sent, and every row expires after 14 days.',\n },\n {\n collection: 'fulfillmentNetworkConnections',\n keyedBy: 'field',\n subjects: ['org'],\n exported: true,\n note: 'EXISTENCE ONLY for the grant — each site’s connection to the merchant’s own ShipBob, ShipMonk, Flexport or Amazon Multi-Channel Fulfillment account (AGL-3634, AGL-3697, AGL-3822): network, status, account name, ShipBob channel, ShipMonk store id, Amazon marketplaces, routing and stock settings, the last stock count by SKU and totals. The sealed access and refresh tokens (for ShipMonk and Flexport, the merchant’s own API key or token, sealed as the access token), the webhook token’s hash and the id of the key that sealed them carry `token` in their names, and ShipMonk’s sealed webhook signing secret carries `secret` in its name; all are redacted (see redactSecrets). The document id is the site id and the network.',\n },\n {\n collection: 'fulfillmentNetworkOrders',\n keyedBy: 'field',\n subjects: ['org'],\n exported: true,\n note: 'Each paid order’s hand-off to a fulfillment network (AGL-3634): the order number, the lines sent with SKU and quantity, the network’s reference, the parcels read back with carrier, tracking number and status, and why anything was not sent. The merchant’s own fulfillment record of a SHOPPER’s order, so disclosed to the ORG only.',\n },\n {\n collection: 'courierConnections',\n keyedBy: 'field',\n subjects: ['org'],\n exported: true,\n note: 'EXISTENCE ONLY for the keys — each site’s connection to the merchant’s own DoorDash Drive developer account (AGL-3695): provider, the developer and key ids of its live and test keys, each key’s last test result, the store’s courier phone and note, and the member who connected it. The sealed signing secrets carry `secret` and the webhook token’s hash carries `token` in their names and are redacted (see redactSecrets); the document id is the site id and the provider.',\n },\n {\n collection: 'courierDeliveries',\n keyedBy: 'field',\n subjects: ['org'],\n exported: true,\n note: 'Each local-delivery order’s courier (AGL-3695): the quote offered, the run booked and the runs before it — the courier’s reference, status, fee, arrival estimates, tracking link and any cancellation reason, and the member who booked it. No address or phone is stored here (they stay on the order); the merchant’s own record of a SHOPPER’s delivery, so disclosed to the ORG only.',\n },\n {\n collection: 'marketplaceConnections',\n keyedBy: 'field',\n subjects: ['org'],\n exported: true,\n note: 'EXISTENCE ONLY for the grant — each site’s connection to the merchant’s own Amazon, eBay, Etsy, TikTok Shop, Walmart or Faire seller account (AGL-3638): marketplace, status, account name, the marketplace sites the account sells on, the listing and order settings, the order cursor and the last sync’s counts. The sealed access and refresh tokens and the id of the key that sealed them carry `token` in their names and are redacted (see redactSecrets); the account’s marketplace ids are disclosed as stored. The document id is the site id and the marketplace. Its `log` and `listingState` subcollections (what each listing was last sent) are not read by this export.',\n },\n {\n collection: 'marketplaceOrders',\n keyedBy: 'field',\n subjects: ['org'],\n exported: true,\n note: 'Each marketplace order the store imported (AGL-3638): the marketplace and its order id, the store order it became, which order line each marketplace line became, the marketplace’s fees, and each shipment confirmed back with carrier and tracking number. The merchant’s own record of a marketplace BUYER’s order, so disclosed to the ORG only; the buyer’s name and address are on the store’s order, not here.',\n },\n {\n collection: 'inventorySyncConnections',\n keyedBy: 'field',\n subjects: ['org'],\n exported: true,\n note: 'EXISTENCE ONLY for the keys — each site’s connection to the merchant’s own Cin7 Core, inFlow, Brightpearl or Ordoro account (AGL-3642): system, status, account name, the stock, product and order settings (location, the customer or Ordoro sales channel orders are recorded under, the tax rule), the last count by SKU, the last syncs’ counts, cursors and totals. The sealed API keys or grant and the id of the key that sealed them carry `credential` in their names and are redacted (see redactSecrets); Brightpearl’s account code and datacenter are disclosed as stored. The document id is the site id. Its `log` subcollection is not read by this export.',\n },\n {\n collection: 'inventorySyncOrders',\n keyedBy: 'field',\n subjects: ['org'],\n exported: true,\n note: 'Each paid order’s hand-off to the connected inventory system (AGL-3642): the order number and our reference, the lines sent with SKU, quantity and price, the buyer’s name, email address and shipping address as sent, the totals, the system’s id and number for it, attempts and why it was not sent. The merchant’s own record of a SHOPPER’s order, so disclosed to the ORG only.',\n },\n {\n collection: 'inventorySyncProducts',\n keyedBy: 'field',\n subjects: ['org'],\n exported: true,\n note: 'Each product linked between the store and the connected inventory system (AGL-3642): the system’s product id, the SKU, the version last synced, the store’s product and variant ids, and why it was not synced. Catalog data, no person’s.',\n },\n {\n collection: 'podConnections',\n keyedBy: 'field',\n subjects: ['org'],\n exported: true,\n note: 'EXISTENCE ONLY for the tokens — each site’s connection to the merchant’s own Printful store or Printify shop (AGL-3641): service, store id and name, currency, whether orders are sent automatically or held for review, whether prices sync, how shipments are heard about, and the last error. The sealed API token and webhook token and the id of the key that sealed them carry `token` in their names and are redacted (see redactSecrets).',\n },\n {\n collection: 'podProductLinks',\n keyedBy: 'field',\n subjects: ['org'],\n exported: true,\n note: 'Each product imported from a print-on-demand service (AGL-3641): the service’s product and variant ids, the store product and variants it became, names, SKUs, what the service charges for each variant and the store’s price. Catalog data, no person’s.',\n },\n {\n collection: 'podOrders',\n keyedBy: 'field',\n subjects: ['org'],\n exported: true,\n note: 'The part of each paid order a print-on-demand service makes and ships (AGL-3641): the store order and its number, the service’s order id, the lines sent with quantity and cost, its status, costs, attempts and errors, and each parcel’s carrier and tracking number. No buyer name, address or email is kept here; they are on the store’s order. The merchant’s own record of a SHOPPER’s order, so disclosed to the ORG only.',\n },\n {\n collection: 'deliveryAppStores',\n keyedBy: 'field',\n subjects: ['org'],\n exported: true,\n note: 'Each site’s link to the merchant’s own DoorDash, Uber Eats or Grubhub store (AGL-3644): the service, the store id it shows the merchant, the settings (accept automatically, prep minutes), the items matched to products and those waiting, and the menu’s last send. No credential is stored: every call is signed with the deployment’s partner credentials.',\n },\n {\n collection: 'deliveryAppOrders',\n keyedBy: 'field',\n subjects: ['org'],\n exported: true,\n note: 'Each order a delivery service sent a site’s register (AGL-3644): the service and its order id and short code, the items with their options and instructions, the totals and refunds, where it stood at the counter, and the store order it became. The buyer appears by first name and last initial only. The merchant’s own record of a delivery BUYER’s order, so disclosed to the ORG only.',\n },\n {\n collection: 'zapierHooks',\n keyedBy: 'field',\n subjects: ['org'],\n exported: true,\n note: 'Each Zapier REST hook a site’s API key subscribed (AGL-3643): the site, the events, the API key’s public id and name, delivery counts and dates. EXISTENCE ONLY for the URL — `targetUrl` is the capability Zapier minted for the Zap, a bearer URL whose secret is in its path, so it is redacted by name (see redactSecrets).',\n },\n {\n collection: 'zapierHookDeliveries',\n keyedBy: 'field',\n subjects: ['org'],\n exported: true,\n note: 'Markers that one event reached one Zapier hook, so a retry does not post it twice (AGL-3643): the hook, the status and the time. The document id is a SHA-256 of the event and hook, an identifier rather than a verifier of any credential; no customer data, and a TTL policy removes each row.',\n },\n {\n collection: 'ssoDomains',\n keyedBy: 'field',\n subjects: ['org'],\n exported: true,\n note: 'The public SSO routing document: domain, GCIP tenant and provider ids, IdP display name. `token` is redacted.',\n },\n {\n collection: CONSOLE_DOMAINS_COLLECTION,\n keyedBy: 'field',\n subjects: ['org'],\n exported: true,\n note: 'Custom console domain claims the org holds.',\n },\n {\n collection: 'apiIdempotency',\n keyedBy: 'field',\n subjects: ['org'],\n exported: false,\n note: 'Replay keys. The document id is a SHA-256 of the org id and the caller’s Idempotency-Key, and the row names a record id and nothing about a person. Disclosed as a COUNT so the export does not silently omit a collection the erasure destroys.',\n },\n {\n collection: 'stripeCustomers',\n keyedBy: 'field',\n subjects: ['org'],\n exported: true,\n note: 'The local reverse index: which Stripe customer id belongs to this workspace. An identifier the customer already holds on their own invoices.',\n },\n {\n collection: 'orgSlugs',\n keyedBy: 'field',\n subjects: ['org'],\n exported: true,\n note: 'Every workspace name the org has ever held, rename tombstones included — naming history is a fact about them.',\n },\n {\n collection: 'publisherHandles',\n keyedBy: 'field',\n subjects: ['org'],\n exported: true,\n note: 'Marketplace handle reservations and rename tombstones (AGL-1970).',\n },\n {\n collection: 'publisherProfiles',\n keyedBy: 'doc-id',\n subjects: ['org'],\n exported: true,\n note: 'The public publisher identity — handle, display name, bio, website, avatar, publisher agreement. `stripeAccountId` is the org’s own payout destination and is disclosed; nothing here is a credential.',\n },\n {\n collection: 'marketplaceListings',\n keyedBy: 'field',\n subjects: ['org'],\n exported: true,\n note: 'Listings the org published. These deliberately OUTLIVE an erasure (AGL-1448 Tier 3) — which makes disclosing them on an access request more important, not less: they are the part that does not go away.',\n },\n {\n collection: 'supportTickets',\n keyedBy: 'field',\n subjects: ['org', 'user'],\n exported: true,\n note: 'For an ORG: every thread and its messages. For a PERSON: only the messages they themselves authored — a thread has other authors, and handing a member the whole conversation would disclose colleagues’ words to them.',\n },\n // ── Retained past erasure on statutory grounds ────────────────────────\n {\n collection: 'platformRevenue',\n keyedBy: 'field',\n subjects: ['org'],\n exported: true,\n note: 'Per-transaction tax filing records. GDPR Art. 17(3)(b) exempts them from ERASURE (AGL-1811); it does not exempt them from ACCESS. If a transaction of theirs is in there it is disclosable — and the export is the only place a customer can see what survived their erasure.',\n },\n {\n collection: 'storefrontTaxCollected',\n keyedBy: 'field',\n subjects: ['org'],\n exported: true,\n note: 'Sales tax charged to shoppers on the org’s storefront (AGL-1904). Same statutory retention, same access obligation.',\n },\n // ── Records of our own handling ───────────────────────────────────────\n {\n collection: 'adminAudit',\n keyedBy: 'field',\n subjects: ['org', 'user'],\n exported: false,\n note: 'STAFF-ONLY. Rows name the staff actor, carry override reasons and internal notes, and are the record of our handling rather than data about the subject. `/admin/audit` → Export CSV serves a request that specifically asks for them, after a staff read.',\n },\n]\n\n/** Sources this export deliberately withholds, with the reason. */\nexport const EXPORT_WITHHELD = PERSONAL_DATA_SOURCES.filter((s) => !s.exported)\n\n/** Collection names the export claims to cover, for the coverage guard. */\nexport const EXPORT_COVERED_COLLECTIONS: readonly string[] =\n PERSONAL_DATA_SOURCES.map((source) => source.collection)\n\n/**\n * ## WHY THIS IS SHAPE-DRIVEN AND NOT A LIST OF NAMES\n *\n * It used to be a list of names, anchored on separators:\n *\n * /(^|[._-])(secret|token|password|apikey|…)($|[._-])/i\n *\n * That pattern requires a `.`, `_`, `-`, or a string boundary on both sides\n * of the word, so it reads `password_hash` and `webhooks.secret` and misses\n * `supplierToken` and `passwordScrypt` — the exact two fields the AGL-1881\n * review found leaving the platform in a customer's own export file. One is\n * a live bearer credential a supplier authenticates with; the other is a\n * shopper's scrypt password hash. **A denylist of names is always one naming\n * convention behind, and camelCase is the proof.**\n *\n * So the name half is now tokenized rather than separator-anchored, and it is\n * no longer the only gate. Three run, in order, and any of them redacts:\n *\n * * **(A) The name, tokenized.** `supplierToken` splits to\n * `['supplier','token']` the same way `supplier_token` and\n * `SUPPLIER-TOKEN` do, so the convention stops mattering. A name matches\n * on a single secret word or on an adjacent pair (`client secret`).\n * * **(B) The VALUE's shape, whatever the key is called.** This is the half\n * that does not need to know the schema: a PEM block, a JWT, a vendor\n * prefix (`sk_live_`, `whsec_`, `ghp_`, `AKIA`…), a `$2b$`/`$argon2` hash,\n * this repo's own `salt:hash` scrypt form, or an opaque high-entropy\n * string is a credential whether the field is called `supplierToken`,\n * `t`, or `notes`. **Gate B is the \"redact unless known safe\" default**:\n * a credential-shaped string is withheld unless its key names it as a\n * content identifier (see {@link SAFE_VALUE_WORDS}).\n * * **(C) Credentials carried inside a URL.** `?token=`, `?sig=`,\n * `?key=`, and `https://user:pass@host` are stripped from the URL rather\n * than the URL being blanked, because a media object's `url` field really\n * does carry a live Firebase download token in a query parameter, and the\n * location of the customer's own file is exactly what an access request\n * asks for. Precision beats blanking here.\n *\n * ### Why the KEY half is still deny-based, when the value half is not\n *\n * A true allowlist of field names is not available on this data. The export\n * walks documents whose field names our customers author — canvas nodes,\n * dataset records, custom metadata, form submissions — so an allowlist of\n * names would blank most of the file and turn an access request into a\n * non-answer, which is the under-disclosure this feature exists to stop. The\n * allowlist therefore lives where the vocabulary IS ours: value shapes. An\n * unrecognized VALUE shape is passed through (it is prose, a number, a date,\n * a customer's own content); an unrecognized value shape that looks like a\n * credential is withheld. That is the inversion the review asked for, put\n * where it can hold.\n *\n * ### The one deliberate carve-out\n *\n * `token`/`tokens` is also a metering word — `tokensUsed: 14203` on an\n * assist exchange is a usage count, and blanking it would blank a real\n * disclosure. A credential is a STRING; a count is a number. So a numeric or\n * boolean value under a name that matched only on a metering word passes\n * through. Nothing else in the vocabulary gets that treatment.\n *\n * A false positive costs a redacted field in an export. A false negative\n * costs a credential. Everything below is tuned in that direction.\n */\n\n/**\n * Split a field name into lowercase word tokens, convention-independently.\n *\n * `supplierToken`, `supplier_token`, `SUPPLIER-TOKEN` and `Supplier.Token`\n * all become `['supplier','token']`. Acronym runs split at the last capital\n * (`SSOTokenId` → `['sso','token','id']`) and digits break a word\n * (`apiKey2` → `['api','key','2']`), so a numbered field cannot hide.\n */\nexport function fieldNameTokens(key: unknown): string[] {\n return String(key ?? '')\n .replace(/[^A-Za-z0-9]+/g, ' ')\n .replace(/([a-z0-9])([A-Z])/g, '$1 $2')\n .replace(/([A-Z]+)([A-Z][a-z])/g, '$1 $2')\n .replace(/([A-Za-z])([0-9])/g, '$1 $2')\n .toLowerCase()\n .split(' ')\n .filter(Boolean)\n}\n\n/**\n * Words that make a field a credential on their own.\n *\n * Deliberately short. Every entry here is a word that has no innocent\n * meaning on a stored VALUE — `hmac`, `scrypt` and `bcrypt` name the\n * algorithm that produced a verifier, `passphrase` and `credential` name the\n * thing itself. `key` is NOT here and must never be: `publicKey` on a\n * passkey and `sortKey` on a dataset are not secrets, which is what\n * {@link SECRET_WORD_PAIRS} is for.\n */\nconst SECRET_WORDS = new Set([\n 'secret',\n 'secrets',\n 'password',\n 'passwords',\n 'passwd',\n 'pwd',\n 'passphrase',\n 'token',\n 'tokens',\n 'credential',\n 'credentials',\n 'apikey',\n 'privatekey',\n 'scrypt',\n 'bcrypt',\n 'argon',\n 'argon2',\n 'pbkdf2',\n 'hmac',\n 'otp',\n 'totp',\n 'authorization',\n])\n\n/**\n * Words that are a credential only next to another word.\n *\n * `key` is meaningless alone and decisive after `api`, `private`, `signing`,\n * `secret`, `session` or `encryption`; `token` after `access`/`refresh` is\n * already caught by {@link SECRET_WORDS} and listed here anyway so the pair\n * table reads as the complete statement of the rule.\n */\nconst SECRET_WORD_PAIRS = new Set([\n 'api key',\n 'private key',\n 'signing key',\n 'secret key',\n 'session key',\n 'encryption key',\n 'client secret',\n 'access token',\n 'refresh token',\n 'webhook secret',\n 'shared secret',\n // A Stripe payment link is a live bearer URL: whoever holds it can pay.\n // Named in the AGL-1443 inventory of what the old dump actually carried.\n 'payment link',\n // A Zapier REST hook's `targetUrl` (AGL-3643) is the same kind of bearer\n // URL: whoever holds it can post into the Zap, and its secret is in the\n // path, which the URL scrub does not read.\n 'target url',\n 'auth code',\n 'recovery code',\n 'backup code',\n])\n\n/**\n * The metering carve-out — see the header. A name that matched ONLY on one\n * of these, over a number or a boolean, is a count and is disclosed.\n */\nconst METERING_WORDS = new Set(['token', 'tokens'])\n\n/**\n * Key words that mark a high-entropy value as a content identifier rather\n * than a credential, defusing gate (B) only.\n *\n * A SHA-256 has exactly the shape of a 64-character secret and is not one.\n * Gate (A) is never defused by this list: `apiKeyId` is redacted, because\n * the name says credential and an id of a credential is not worth the\n * argument.\n */\nconst SAFE_VALUE_WORDS = new Set([\n // A passkey's `publicKey` is public by definition and has exactly the shape\n // of a secret. Safe to list because gate (A) is never defused here — a\n // `publicToken` is still redacted, on the strength of `token`.\n 'public',\n 'id',\n 'ids',\n 'uid',\n 'uuid',\n 'guid',\n 'hash',\n 'hashes',\n 'sha',\n 'sha1',\n 'sha256',\n 'sha512',\n 'md5',\n 'crc',\n 'crc32c',\n 'etag',\n 'checksum',\n 'digest',\n 'fingerprint',\n 'generation',\n 'version',\n 'revision',\n 'slug',\n 'path',\n 'url',\n 'uri',\n 'href',\n 'src',\n 'color',\n 'colour',\n 'hex',\n])\n\n/** Vendor credential prefixes — unambiguous, and worth naming explicitly. */\nconst CREDENTIAL_PREFIXES = [\n 'sk_live_',\n 'sk_test_',\n 'rk_live_',\n 'rk_test_',\n 'whsec_',\n 'ghp_',\n 'gho_',\n 'ghu_',\n 'ghs_',\n 'ghr_',\n 'github_pat_',\n 'xoxb-',\n 'xoxp-',\n 'xapp-',\n 'xoxa-',\n 'AKIA',\n 'ASIA',\n 'AIza',\n 'ya29.',\n 'SG.',\n 're_',\n 'shpat_',\n 'shpss_',\n 'npm_',\n 'glpat-',\n 'sq0csp-',\n 'sq0atp-',\n]\n\n/** A PEM block that carries a private key or a certificate request. */\nconst PEM_PATTERN = /-----BEGIN [A-Z0-9 ]*PRIVATE KEY-----/\n\n/** Modular crypt format: bcrypt, argon2, scrypt, PBKDF2, sha512-crypt. */\nconst CRYPT_HASH_PATTERN = /^\\$(2[abxy]?|argon2[a-z]*|scrypt|pbkdf2[^$]*|[56])\\$/\n\n/** This repo's own scrypt storage form: `{16+ hex salt}:{32+ hex hash}`. */\nconst SALTED_HASH_PATTERN = /^[0-9a-f]{16,}:[0-9a-f]{32,}$/i\n\n/** A JSON Web Token — three base64url segments. */\nconst JWT_PATTERN = /^[A-Za-z0-9_-]{8,}\\.[A-Za-z0-9_-]{8,}\\.[A-Za-z0-9_-]{8,}$/\n\n/** A UUID, which is an identifier far more often than it is a credential. */\nconst UUID_PATTERN =\n /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i\n\n/** Charset of an opaque credential: base64, base64url or hex, nothing else. */\nconst OPAQUE_CHARSET = /^[A-Za-z0-9_+/=-]+$/\n\n/**\n * The window an opaque secret lives in.\n *\n * Below 32 characters the false-positive rate against ordinary identifiers is\n * not worth it and the value is too short to be a modern credential anyway.\n * Above 512 it is customer CONTENT — a base64 image, a compressed canvas\n * blob, an encoded document — and blanking those is the \"gutting the export\"\n * failure this file is explicitly trying not to commit.\n */\nconst OPAQUE_MIN_LENGTH = 32\nconst OPAQUE_MAX_LENGTH = 512\n\n/** Shannon entropy per character. Prose and slugs sit well below this. */\nconst OPAQUE_MIN_ENTROPY = 3.2\n\nfunction shannonEntropyPerChar(text: string): number {\n const counts = new Map<string, number>()\n for (const character of text) {\n counts.set(character, (counts.get(character) ?? 0) + 1)\n }\n let entropy = 0\n for (const count of counts.values()) {\n const p = count / text.length\n entropy -= p * Math.log2(p)\n }\n return entropy\n}\n\n/**\n * Does this STRING look like a credential, ignoring what it is called?\n *\n * Exported because the guard spec asserts each limb independently — a shape\n * detector tested only through the redactor is a detector whose gaps are\n * invisible.\n */\nexport function looksLikeCredential(value: string): boolean {\n const text = value.trim()\n if (!text) return false\n if (PEM_PATTERN.test(text)) return true\n if (CRYPT_HASH_PATTERN.test(text)) return true\n if (SALTED_HASH_PATTERN.test(text)) return true\n if (CREDENTIAL_PREFIXES.some((prefix) => text.startsWith(prefix))) return true\n if (JWT_PATTERN.test(text)) return true\n // Anything with whitespace is prose, and `data:` is an inline asset. Both\n // are customer content, and neither is a bearer credential.\n if (/\\s/.test(text)) return false\n if (/^data:/i.test(text)) return false\n if (UUID_PATTERN.test(text)) return false\n if (text.length < OPAQUE_MIN_LENGTH || text.length > OPAQUE_MAX_LENGTH) {\n return false\n }\n if (!OPAQUE_CHARSET.test(text)) return false\n return shannonEntropyPerChar(text) >= OPAQUE_MIN_ENTROPY\n}\n\n/** What a stripped URL parameter reads as, so the removal is visible. */\nexport const REDACTED_URL_PARAM = 'REDACTED'\n\n/** Query parameters that carry a bearer credential inside a URL. */\nconst URL_CREDENTIAL_PARAMS = new Set([\n 'token',\n 'access_token',\n 'accesstoken',\n 'id_token',\n 'refresh_token',\n 'auth',\n 'authorization',\n 'apikey',\n 'api_key',\n 'key',\n 'sig',\n 'signature',\n 'secret',\n 'password',\n 'downloadtoken',\n])\n\n/**\n * Strip the credential out of a URL, keeping the URL.\n *\n * A media object's `url` is `…/o/{path}?alt=media&token={uuid}` — a live,\n * unauthenticated download credential, in a field an access request has a\n * real interest in. Blanking the whole value would tell the customer nothing\n * about where their own file is; keeping it whole hands out the credential.\n * So the parameter is replaced and the rest survives. `user:pass@` userinfo\n * goes the same way.\n *\n * Returns the input unchanged when it is not a URL — every non-URL string\n * has already been decided by the shape gate.\n */\nexport function scrubUrlCredentials(value: string): string {\n if (!/^https?:\\/\\//i.test(value)) return value\n let url: URL\n try {\n url = new URL(value)\n } catch {\n return value\n }\n let changed = false\n if (url.username || url.password) {\n url.username = ''\n url.password = ''\n changed = true\n }\n for (const name of Array.from(url.searchParams.keys())) {\n if (!URL_CREDENTIAL_PARAMS.has(name.toLowerCase())) continue\n if (!url.searchParams.get(name)) continue\n url.searchParams.set(name, REDACTED_URL_PARAM)\n changed = true\n }\n return changed ? url.toString() : value\n}\n\n/**\n * Gate (A): does the NAME say credential?\n *\n * `value` is consulted only for the metering carve-out described in the\n * header — a `tokensUsed: 14203` is a count, not a credential.\n */\nexport function secretByName(key: unknown, value?: unknown): boolean {\n const tokens = fieldNameTokens(key)\n if (!tokens.length) return false\n const hits = tokens.filter((token) => SECRET_WORDS.has(token))\n for (let index = 0; index + 1 < tokens.length; index += 1) {\n if (SECRET_WORD_PAIRS.has(`${tokens[index]} ${tokens[index + 1]}`)) {\n return true\n }\n }\n if (!hits.length) return false\n // Matched only on a metering word, over a number or a boolean: a count.\n const metering = hits.every((token) => METERING_WORDS.has(token))\n if (metering && (typeof value === 'number' || typeof value === 'boolean')) {\n return false\n }\n return true\n}\n\n/** Gate (B)'s defuser: does the NAME say this high-entropy value is an id? */\nfunction nameMarksContentIdentifier(key: unknown): boolean {\n return fieldNameTokens(key).some((token) => SAFE_VALUE_WORDS.has(token))\n}\n\n/** What a redacted value is replaced with — presence, never content. */\nexport interface RedactedValue {\n redacted: true\n /** Whether a value was actually there, so absence is distinguishable. */\n present: boolean\n reason: 'secret'\n}\n\nconst redactedMarker = (present: boolean): RedactedValue => ({\n redacted: true,\n present,\n reason: 'secret',\n})\n\n/**\n * Deep-copy a document, replacing every credential-shaped value with a\n * presence marker.\n *\n * Presence rather than deletion, because \"we hold a webhook secret for you\"\n * is itself a true and disclosable fact, and a silently-absent key would make\n * the export look like it holds less than it does — the under-disclosure this\n * feature exists to stop, reintroduced by the safety measure.\n *\n * Firestore values that are not plain JSON (Timestamp, GeoPoint, DocumentRef,\n * Buffer) are stringified rather than walked: they carry no nested keys to\n * redact, and `JSON.stringify` on a Timestamp yields `{_seconds,_nanoseconds}`,\n * which is not a date to a human reading the file.\n *\n * Three gates, any of which redacts — the name (A), the value's shape (B),\n * and a credential embedded in a URL (C). See the header above\n * {@link fieldNameTokens} for why the value half is the one that carries the\n * \"withhold unless known safe\" default.\n */\nexport function redactSecrets(value: unknown, key = ''): unknown {\n // (A) The name, tokenized — convention-independent.\n if (key && secretByName(key, value)) {\n return redactedMarker(value !== undefined && value !== null && value !== '')\n }\n if (value === null || value === undefined) return value ?? null\n if (typeof value === 'string') {\n // (C) A URL that carries its credential in a parameter keeps the URL and\n // loses the credential — see {@link scrubUrlCredentials}.\n const scrubbed = scrubUrlCredentials(value)\n if (scrubbed !== value) return scrubbed\n // (B) The shape, whatever the field is called. Withheld unless the name\n // marks it as a content identifier.\n if (looksLikeCredential(value) && !nameMarksContentIdentifier(key)) {\n return redactedMarker(true)\n }\n return value\n }\n if (Array.isArray(value)) return value.map((entry) => redactSecrets(entry, key))\n if (typeof value === 'object') {\n const source = value as Record<string, unknown>\n // Firestore Timestamp — the only non-plain value common enough to be\n // worth rendering rather than dumping.\n if (typeof (source as any).toDate === 'function') {\n try {\n return (source as any).toDate().toISOString()\n } catch {\n return String(value)\n }\n }\n if (typeof (source as any).toBase64 === 'function') return '[binary]'\n const out: Record<string, unknown> = {}\n for (const [childKey, childValue] of Object.entries(source)) {\n out[childKey] = redactSecrets(childValue, childKey)\n }\n return out\n }\n return value\n}\n\nexport interface ExportedDocument {\n /** Document id — omitted where the id is itself a secret (see `apiKeys`). */\n id?: string\n path?: string\n data: unknown\n}\n\nexport interface PersonalDataExport {\n subject: { type: ExportSubject; id: string }\n generatedAt: string\n /**\n * The disclosure decision for every source, copied into the file the\n * customer receives. An export that silently omits a collection is\n * indistinguishable from one that has nothing to report; this makes the two\n * different, and it is what a supervisory authority would ask to see.\n */\n coverage: readonly ExportSourceSpec[]\n /** Collection name → the documents disclosed for it. */\n data: Record<string, ExportedDocument[]>\n /** Collection name → row count, for sources disclosed as a count only. */\n counts: Record<string, number>\n /** Places we hold data that is not in Firestore. */\n elsewhere: readonly string[]\n}\n\n/** Data we hold outside Firestore — named, because a file that omits them reads as complete. */\nconst ELSEWHERE = [\n 'Stripe — customer object, payment methods, invoices. Request from Stripe or ask us.',\n 'Resend — transactional email delivery logs.',\n 'Google Analytics — 14-month retention, not keyed to an account.',\n 'Cloud Storage — uploaded media and avatars, under users/{uid}/, orgs/{orgId}/ and hosts/{hostId}/. Listed by path in this file; the bytes are served by their own URLs.',\n]\n\nfunction sourcesFor(subject: ExportSubject): ExportSourceSpec[] {\n return PERSONAL_DATA_SOURCES.filter((source) => source.subjects.includes(subject))\n}\n\n/** Read a top-level collection bounded by a field equal to the subject id. */\nasync function readByField(\n db: any,\n collection: string,\n field: string,\n id: string,\n options: { includeIds?: boolean } = {},\n): Promise<ExportedDocument[]> {\n const rows = await db.collection(collection).where(field, '==', id).get()\n return rows.docs.map((doc: any) => ({\n // `apiKeys` document ids are the SHA-256 of the token: an id handed back\n // is a verifier handed back, so the caller opts in rather than out.\n ...(options.includeIds === false ? {} : { id: doc.id }),\n data: redactSecrets(doc.data()),\n }))\n}\n\n/** Read one document by id, or nothing when it does not exist. */\nasync function readDoc(\n db: any,\n collection: string,\n id: string,\n): Promise<ExportedDocument[]> {\n const snapshot = await db.collection(collection).doc(id).get()\n if (!snapshot.exists) return []\n return [{ id, data: redactSecrets(snapshot.data()) }]\n}\n\n/**\n * Walk a document and everything beneath it.\n *\n * `listCollections()` rather than a hard-coded subcollection list, for the\n * same reason the erasure uses `recursiveDelete`: a subcollection added next\n * quarter is reached without anybody remembering this file exists. Bounded by\n * `maxDepth` because the tree is customer-shaped and an unbounded recursion\n * inside a request is how AGL-1455 lost erasures to a 60-second timeout.\n */\nasync function readSubtree(\n db: any,\n ref: any,\n depth = 0,\n maxDepth = 4,\n): Promise<ExportedDocument[]> {\n const snapshot = await ref.get()\n const found: ExportedDocument[] = []\n if (snapshot.exists) {\n found.push({ path: ref.path, data: redactSecrets(snapshot.data()) })\n }\n if (depth >= maxDepth) return found\n const children = (await ref.listCollections?.()) ?? []\n for (const child of children) {\n const rows = await child.get()\n for (const doc of rows.docs) {\n found.push(...(await readSubtree(db, doc.ref, depth + 1, maxDepth)))\n }\n }\n return found\n}\n\nexport interface ExportOptions {\n /** Injectable for tests; defaults to the admin app's Firestore. */\n firestore?: any\n /** Stops a pathological workspace from building an unbounded file. */\n maxDepth?: number\n}\n\n/**\n * Everything we hold about a PERSON, as the person themselves may ask for it\n * (GDPR Art. 15 and Art. 20).\n *\n * Scoped to them, never to the workspaces they belong to. Their membership row\n * in each org is theirs; the roster around it is their colleagues'. An org\n * export is a separate call with a separate authorization, which is the whole\n * reason the two are different functions rather than one with a flag.\n */\nexport async function exportUserData(\n uid: string,\n options: ExportOptions = {},\n): Promise<PersonalDataExport> {\n const db = options.firestore ?? firestoreApp()\n const data: Record<string, ExportedDocument[]> = {}\n const counts: Record<string, number> = {}\n\n // The profile tree — `users/{uid}` and every subcollection under it.\n data['users'] = await readSubtree(\n db,\n db.collection('users').doc(uid),\n 0,\n options.maxDepth ?? 4,\n )\n\n // The public identity, which the user tree does not reach (AGL-1970).\n data['profiles'] = await readDoc(db, 'profiles', uid)\n\n // ONLY the messages this person wrote. The threads belong to a workspace\n // and carry other people's words; handing a member the whole conversation\n // to answer their own access request is a disclosure nobody consented to.\n const messages = await db\n .collectionGroup('messages')\n .where('authorId', '==', uid)\n .get()\n .catch(() => ({ docs: [] as any[] }))\n data['supportTickets'] = messages.docs.map((doc: any) => ({\n path: doc.ref.path,\n data: redactSecrets(doc.data()),\n }))\n\n return {\n subject: { type: 'user', id: uid },\n generatedAt: new Date().toISOString(),\n coverage: sourcesFor('user'),\n data,\n counts,\n elsewhere: ELSEWHERE,\n }\n}\n\n/**\n * Everything we hold about a WORKSPACE, for its owner (Art. 15 for the\n * personal data in it, and the \"export before you delete\" the Delete tab,\n * DPA §11 and Terms §13.3 have been promising).\n *\n * Every top-level collection the erasure sweeps by `orgId` is read here by the\n * same bound, so the two see the same rows. Where a source is withheld, its\n * COUNT is still reported: a collection that appears in neither the data nor\n * the counts is indistinguishable from one that does not exist, and that\n * ambiguity is what makes an incomplete export undetectable.\n */\nexport async function exportOrgData(\n orgId: string,\n options: ExportOptions = {},\n): Promise<PersonalDataExport> {\n const db = options.firestore ?? firestoreApp()\n const data: Record<string, ExportedDocument[]> = {}\n const counts: Record<string, number> = {}\n\n data['orgs'] = await readSubtree(\n db,\n db.collection('orgs').doc(orgId),\n 0,\n options.maxDepth ?? 4,\n )\n\n // Sites: the routing entry and the whole tree for each.\n const hosts = await db.collection('hosts').where('orgId', '==', orgId).get()\n const hostDocs: ExportedDocument[] = []\n const routing: ExportedDocument[] = []\n const outbox: ExportedDocument[] = []\n for (const host of hosts.docs) {\n hostDocs.push(...(await readSubtree(db, host.ref, 0, options.maxDepth ?? 4)))\n routing.push(...(await readDoc(db, 'hostIndex', host.id)))\n // Keyed by `hostId`, not `orgId`, so it is read here rather than in the\n // field-keyed loop below — the same bound `eraseHostSupplierDeliveries`\n // sweeps by, so the export and the erasure see the same rows.\n outbox.push(...(await readByField(db, 'supplierDeliveries', 'hostId', host.id)))\n }\n data['hosts'] = hostDocs\n data['hostIndex'] = routing\n data['supplierDeliveries'] = outbox\n\n // The field-keyed collections a path cascade is blind to — the ones the\n // erasure had to grow a sweep for, one issue at a time.\n for (const collection of [\n 'ssoDomains',\n CONSOLE_DOMAINS_COLLECTION,\n 'stripeCustomers',\n 'orgSlugs',\n 'publisherHandles',\n 'platformRevenue',\n 'storefrontTaxCollected',\n ]) {\n data[collection] = await readByField(db, collection, 'orgId', orgId)\n }\n\n // Existence, never the credential — and not the id either, which is the\n // token's SHA-256.\n data['apiKeys'] = await readByField(db, 'apiKeys', 'orgId', orgId, {\n includeIds: false,\n })\n\n // Which mailboxes the reps connected, never the grant behind one: the\n // tokens are withheld by `redactSecrets` like every other read here. The id\n // is the mailbox's own, so unlike `apiKeys` it stays.\n data['outreachMailboxCredentials'] = await readByField(\n db,\n 'outreachMailboxCredentials',\n 'orgId',\n orgId,\n )\n\n // Where each short tracking link the org's sequences sent points (AGL-3297).\n data['outreachLinks'] = await readByField(db, 'outreachLinks', 'orgId', orgId)\n\n // Shipping (AGL-3612), tax services (AGL-3631) and marketing platforms\n // (AGL-3639): top-level and keyed by `orgId`, the bound their\n // `orgKeyedCollections` erasure sweeps by. The sealed credentials are\n // withheld by `redactSecrets` on their names.\n for (const collection of [\n 'shippingTrackers',\n 'shippingQuoteCache',\n 'taxEngineConnections',\n 'taxEngineProductCodes',\n 'taxEngineExemptions',\n 'taxEngineTransactions',\n 'marketingPlatformConnections',\n 'marketingPlatformEvents',\n 'adConversionConnections',\n 'adConversionConsents',\n 'adConversionEvents',\n 'fulfillmentNetworkConnections',\n 'fulfillmentNetworkOrders',\n 'courierConnections',\n 'courierDeliveries',\n 'marketplaceConnections',\n 'marketplaceOrders',\n 'inventorySyncConnections',\n 'inventorySyncOrders',\n 'inventorySyncProducts',\n 'podConnections',\n 'podProductLinks',\n 'podOrders',\n 'deliveryAppStores',\n 'deliveryAppOrders',\n ]) {\n data[collection] = await readByField(db, collection, 'orgId', orgId)\n }\n\n data['publisherProfiles'] = await readDoc(db, 'publisherProfiles', orgId)\n data['marketplaceListings'] = await readByField(\n db,\n 'marketplaceListings',\n 'profileId',\n orgId,\n )\n\n // Support threads WITH their messages — for an org the whole thread is the\n // org's own record, unlike the person-scoped export above.\n const tickets = await db\n .collection('supportTickets')\n .where('orgId', '==', orgId)\n .get()\n const ticketDocs: ExportedDocument[] = []\n for (const ticket of tickets.docs) {\n ticketDocs.push(...(await readSubtree(db, ticket.ref, 0, 2)))\n }\n data['supportTickets'] = ticketDocs\n\n // Withheld, but counted — see the note on this function.\n const replayKeys = await db\n .collection('apiIdempotency')\n .where('orgId', '==', orgId)\n .get()\n counts['apiIdempotency'] = replayKeys.docs.length\n\n return {\n subject: { type: 'org', id: orgId },\n generatedAt: new Date().toISOString(),\n coverage: sourcesFor('org'),\n data,\n counts,\n elsewhere: ELSEWHERE,\n }\n}\n\n/** A filename a human can file: subject, id and the date it was produced. */\nexport function exportFilename(exported: PersonalDataExport): string {\n const day = exported.generatedAt.slice(0, 10)\n return `aglyn-${exported.subject.type}-data-${exported.subject.id}-${day}.json`\n}\n\nexport default exportUserData\n"],"names":["firebaseAdmin","CONSOLE_DOMAINS_COLLECTION","firestoreApp","app","firestore","PERSONAL_DATA_SOURCES","collection","keyedBy","subjects","exported","note","EXPORT_WITHHELD","filter","s","EXPORT_COVERED_COLLECTIONS","map","source","fieldNameTokens","key","String","replace","toLowerCase","split","Boolean","SECRET_WORDS","Set","SECRET_WORD_PAIRS","METERING_WORDS","SAFE_VALUE_WORDS","CREDENTIAL_PREFIXES","PEM_PATTERN","CRYPT_HASH_PATTERN","SALTED_HASH_PATTERN","JWT_PATTERN","UUID_PATTERN","OPAQUE_CHARSET","OPAQUE_MIN_LENGTH","OPAQUE_MAX_LENGTH","OPAQUE_MIN_ENTROPY","shannonEntropyPerChar","text","counts","Map","character","set","get","entropy","count","values","p","length","Math","log2","looksLikeCredential","value","trim","test","some","prefix","startsWith","REDACTED_URL_PARAM","URL_CREDENTIAL_PARAMS","scrubUrlCredentials","url","URL","changed","username","password","name","Array","from","searchParams","keys","has","toString","secretByName","tokens","hits","token","index","metering","every","nameMarksContentIdentifier","redactedMarker","present","redacted","reason","redactSecrets","undefined","scrubbed","isArray","entry","toDate","toISOString","toBase64","out","childKey","childValue","Object","entries","ELSEWHERE","sourcesFor","subject","includes","readByField","db","field","id","options","rows","where","docs","doc","includeIds","data","readDoc","snapshot","exists","readSubtree","ref","depth","maxDepth","found","push","path","children","listCollections","child","exportUserData","uid","messages","collectionGroup","catch","type","generatedAt","Date","coverage","elsewhere","exportOrgData","orgId","hosts","hostDocs","routing","outbox","host","tickets","ticketDocs","ticket","replayKeys","exportFilename","day","slice"],"mappings":";AAAA;;;;;;;;;;;;;;;CAeC,GAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA0DC,GAED,SAASA,aAAa,QAAQ,sBAAkB;AAChD,SAASC,0BAA0B,QAAQ,uBAAmB;AAE9D,MAAMC,eAAe,IAAMF,cAAcG,GAAG,GAAGC,SAAS;AAiCxD;;;;;;CAMC,GACD,OAAO,MAAMC,wBAAqD;IAChE,yEAAyE;IACzE;QACEC,YAAY;QACZC,SAAS;QACTC,UAAU;YAAC;SAAO;QAClBC,UAAU;QACVC,MAAM;IACR;IACA;QACEJ,YAAY;QACZC,SAAS;QACTC,UAAU;YAAC;SAAO;QAClBC,UAAU;QACVC,MAAM;IACR;IACA,yEAAyE;IACzE;QACEJ,YAAY;QACZC,SAAS;QACTC,UAAU;YAAC;SAAM;QACjBC,UAAU;QACVC,MAAM;IACR;IACA;QACEJ,YAAY;QACZC,SAAS;QACTC,UAAU;YAAC;SAAM;QACjBC,UAAU;QACVC,MAAM;IACR;IACA;QACEJ,YAAY;QACZC,SAAS;QACTC,UAAU;YAAC;SAAM;QACjBC,UAAU;QACVC,MAAM;IACR;IACA;QACEJ,YAAY;QACZC,SAAS;QACTC,UAAU;YAAC;SAAM;QACjBC,UAAU;QACVC,MAAM;IACR;IACA,yEAAyE;IACzE;QACEJ,YAAY;QACZC,SAAS;QACTC,UAAU;YAAC;SAAM;QACjBC,UAAU;QACVC,MAAM;IACR;IACA;QACEJ,YAAY;QACZC,SAAS;QACTC,UAAU;YAAC;SAAM;QACjBC,UAAU;QACVC,MAAM;IACR;IACA;QACEJ,YAAY;QACZC,SAAS;QACTC,UAAU;YAAC;SAAM;QACjBC,UAAU;QACVC,MAAM;IACR;IACA;QACEJ,YAAY;QACZC,SAAS;QACTC,UAAU;YAAC;SAAM;QACjBC,UAAU;QACVC,MAAM;IACR;IACA;QACEJ,YAAY;QACZC,SAAS;QACTC,UAAU;YAAC;SAAM;QACjBC,UAAU;QACVC,MAAM;IACR;IACA;QACEJ,YAAY;QACZC,SAAS;QACTC,UAAU;YAAC;SAAM;QACjBC,UAAU;QACVC,MAAM;IACR;IACA;QACEJ,YAAY;QACZC,SAAS;QACTC,UAAU;YAAC;SAAM;QACjBC,UAAU;QACVC,MAAM;IACR;IACA;QACEJ,YAAY;QACZC,SAAS;QACTC,UAAU;YAAC;SAAM;QACjBC,UAAU;QACVC,MAAM;IACR;IACA;QACEJ,YAAY;QACZC,SAAS;QACTC,UAAU;YAAC;SAAM;QACjBC,UAAU;QACVC,MAAM;IACR;IACA;QACEJ,YAAY;QACZC,SAAS;QACTC,UAAU;YAAC;SAAM;QACjBC,UAAU;QACVC,MAAM;IACR;IACA;QACEJ,YAAY;QACZC,SAAS;QACTC,UAAU;YAAC;SAAM;QACjBC,UAAU;QACVC,MAAM;IACR;IACA;QACEJ,YAAY;QACZC,SAAS;QACTC,UAAU;YAAC;SAAM;QACjBC,UAAU;QACVC,MAAM;IACR;IACA;QACEJ,YAAY;QACZC,SAAS;QACTC,UAAU;YAAC;SAAM;QACjBC,UAAU;QACVC,MAAM;IACR;IACA;QACEJ,YAAY;QACZC,SAAS;QACTC,UAAU;YAAC;SAAM;QACjBC,UAAU;QACVC,MAAM;IACR;IACA;QACEJ,YAAY;QACZC,SAAS;QACTC,UAAU;YAAC;SAAM;QACjBC,UAAU;QACVC,MAAM;IACR;IACA;QACEJ,YAAY;QACZC,SAAS;QACTC,UAAU;YAAC;SAAM;QACjBC,UAAU;QACVC,MAAM;IACR;IACA;QACEJ,YAAY;QACZC,SAAS;QACTC,UAAU;YAAC;SAAM;QACjBC,UAAU;QACVC,MAAM;IACR;IACA;QACEJ,YAAY;QACZC,SAAS;QACTC,UAAU;YAAC;SAAM;QACjBC,UAAU;QACVC,MAAM;IACR;IACA;QACEJ,YAAY;QACZC,SAAS;QACTC,UAAU;YAAC;SAAM;QACjBC,UAAU;QACVC,MAAM;IACR;IACA;QACEJ,YAAY;QACZC,SAAS;QACTC,UAAU;YAAC;SAAM;QACjBC,UAAU;QACVC,MAAM;IACR;IACA;QACEJ,YAAY;QACZC,SAAS;QACTC,UAAU;YAAC;SAAM;QACjBC,UAAU;QACVC,MAAM;IACR;IACA;QACEJ,YAAY;QACZC,SAAS;QACTC,UAAU;YAAC;SAAM;QACjBC,UAAU;QACVC,MAAM;IACR;IACA;QACEJ,YAAY;QACZC,SAAS;QACTC,UAAU;YAAC;SAAM;QACjBC,UAAU;QACVC,MAAM;IACR;IACA;QACEJ,YAAY;QACZC,SAAS;QACTC,UAAU;YAAC;SAAM;QACjBC,UAAU;QACVC,MAAM;IACR;IACA;QACEJ,YAAY;QACZC,SAAS;QACTC,UAAU;YAAC;SAAM;QACjBC,UAAU;QACVC,MAAM;IACR;IACA;QACEJ,YAAY;QACZC,SAAS;QACTC,UAAU;YAAC;SAAM;QACjBC,UAAU;QACVC,MAAM;IACR;IACA;QACEJ,YAAY;QACZC,SAAS;QACTC,UAAU;YAAC;SAAM;QACjBC,UAAU;QACVC,MAAM;IACR;IACA;QACEJ,YAAY;QACZC,SAAS;QACTC,UAAU;YAAC;SAAM;QACjBC,UAAU;QACVC,MAAM;IACR;IACA;QACEJ,YAAY;QACZC,SAAS;QACTC,UAAU;YAAC;SAAM;QACjBC,UAAU;QACVC,MAAM;IACR;IACA;QACEJ,YAAY;QACZC,SAAS;QACTC,UAAU;YAAC;SAAM;QACjBC,UAAU;QACVC,MAAM;IACR;IACA;QACEJ,YAAY;QACZC,SAAS;QACTC,UAAU;YAAC;SAAM;QACjBC,UAAU;QACVC,MAAM;IACR;IACA;QACEJ,YAAYL;QACZM,SAAS;QACTC,UAAU;YAAC;SAAM;QACjBC,UAAU;QACVC,MAAM;IACR;IACA;QACEJ,YAAY;QACZC,SAAS;QACTC,UAAU;YAAC;SAAM;QACjBC,UAAU;QACVC,MAAM;IACR;IACA;QACEJ,YAAY;QACZC,SAAS;QACTC,UAAU;YAAC;SAAM;QACjBC,UAAU;QACVC,MAAM;IACR;IACA;QACEJ,YAAY;QACZC,SAAS;QACTC,UAAU;YAAC;SAAM;QACjBC,UAAU;QACVC,MAAM;IACR;IACA;QACEJ,YAAY;QACZC,SAAS;QACTC,UAAU;YAAC;SAAM;QACjBC,UAAU;QACVC,MAAM;IACR;IACA;QACEJ,YAAY;QACZC,SAAS;QACTC,UAAU;YAAC;SAAM;QACjBC,UAAU;QACVC,MAAM;IACR;IACA;QACEJ,YAAY;QACZC,SAAS;QACTC,UAAU;YAAC;SAAM;QACjBC,UAAU;QACVC,MAAM;IACR;IACA;QACEJ,YAAY;QACZC,SAAS;QACTC,UAAU;YAAC;YAAO;SAAO;QACzBC,UAAU;QACVC,MAAM;IACR;IACA,yEAAyE;IACzE;QACEJ,YAAY;QACZC,SAAS;QACTC,UAAU;YAAC;SAAM;QACjBC,UAAU;QACVC,MAAM;IACR;IACA;QACEJ,YAAY;QACZC,SAAS;QACTC,UAAU;YAAC;SAAM;QACjBC,UAAU;QACVC,MAAM;IACR;IACA,yEAAyE;IACzE;QACEJ,YAAY;QACZC,SAAS;QACTC,UAAU;YAAC;YAAO;SAAO;QACzBC,UAAU;QACVC,MAAM;IACR;CACD,CAAA;AAED,iEAAiE,GACjE,OAAO,MAAMC,kBAAkBN,sBAAsBO,MAAM,CAAC,CAACC,IAAM,CAACA,EAAEJ,QAAQ,EAAC;AAE/E,yEAAyE,GACzE,OAAO,MAAMK,6BACXT,sBAAsBU,GAAG,CAAC,CAACC,SAAWA,OAAOV,UAAU,EAAC;AAE1D;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA4DC,GAED;;;;;;;CAOC,GACD,OAAO,SAASW,gBAAgBC,GAAY;IAC1C,OAAOC,OAAOD,cAAAA,MAAO,IAClBE,OAAO,CAAC,kBAAkB,KAC1BA,OAAO,CAAC,sBAAsB,SAC9BA,OAAO,CAAC,yBAAyB,SACjCA,OAAO,CAAC,sBAAsB,SAC9BC,WAAW,GACXC,KAAK,CAAC,KACNV,MAAM,CAACW;AACZ;AAEA;;;;;;;;;CASC,GACD,MAAMC,eAAe,IAAIC,IAAI;IAC3B;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;CACD;AAED;;;;;;;CAOC,GACD,MAAMC,oBAAoB,IAAID,IAAI;IAChC;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA,wEAAwE;IACxE,yEAAyE;IACzE;IACA,yEAAyE;IACzE,wEAAwE;IACxE,2CAA2C;IAC3C;IACA;IACA;IACA;CACD;AAED;;;CAGC,GACD,MAAME,iBAAiB,IAAIF,IAAI;IAAC;IAAS;CAAS;AAElD;;;;;;;;CAQC,GACD,MAAMG,mBAAmB,IAAIH,IAAI;IAC/B,4EAA4E;IAC5E,uEAAuE;IACvE,+DAA+D;IAC/D;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;CACD;AAED,2EAA2E,GAC3E,MAAMI,sBAAsB;IAC1B;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;CACD;AAED,qEAAqE,GACrE,MAAMC,cAAc;AAEpB,wEAAwE,GACxE,MAAMC,qBAAqB;AAE3B,0EAA0E,GAC1E,MAAMC,sBAAsB;AAE5B,iDAAiD,GACjD,MAAMC,cAAc;AAEpB,2EAA2E,GAC3E,MAAMC,eACJ;AAEF,6EAA6E,GAC7E,MAAMC,iBAAiB;AAEvB;;;;;;;;CAQC,GACD,MAAMC,oBAAoB;AAC1B,MAAMC,oBAAoB;AAE1B,wEAAwE,GACxE,MAAMC,qBAAqB;AAE3B,SAASC,sBAAsBC,IAAY;IACzC,MAAMC,SAAS,IAAIC;IACnB,KAAK,MAAMC,aAAaH,KAAM;YACLC;QAAvBA,OAAOG,GAAG,CAACD,WAAW,EAACF,cAAAA,OAAOI,GAAG,CAACF,sBAAXF,cAAyB,KAAK;IACvD;IACA,IAAIK,UAAU;IACd,KAAK,MAAMC,SAASN,OAAOO,MAAM,GAAI;QACnC,MAAMC,IAAIF,QAAQP,KAAKU,MAAM;QAC7BJ,WAAWG,IAAIE,KAAKC,IAAI,CAACH;IAC3B;IACA,OAAOH;AACT;AAEA;;;;;;CAMC,GACD,OAAO,SAASO,oBAAoBC,KAAa;IAC/C,MAAMd,OAAOc,MAAMC,IAAI;IACvB,IAAI,CAACf,MAAM,OAAO;IAClB,IAAIV,YAAY0B,IAAI,CAAChB,OAAO,OAAO;IACnC,IAAIT,mBAAmByB,IAAI,CAAChB,OAAO,OAAO;IAC1C,IAAIR,oBAAoBwB,IAAI,CAAChB,OAAO,OAAO;IAC3C,IAAIX,oBAAoB4B,IAAI,CAAC,CAACC,SAAWlB,KAAKmB,UAAU,CAACD,UAAU,OAAO;IAC1E,IAAIzB,YAAYuB,IAAI,CAAChB,OAAO,OAAO;IACnC,0EAA0E;IAC1E,4DAA4D;IAC5D,IAAI,KAAKgB,IAAI,CAAChB,OAAO,OAAO;IAC5B,IAAI,UAAUgB,IAAI,CAAChB,OAAO,OAAO;IACjC,IAAIN,aAAasB,IAAI,CAAChB,OAAO,OAAO;IACpC,IAAIA,KAAKU,MAAM,GAAGd,qBAAqBI,KAAKU,MAAM,GAAGb,mBAAmB;QACtE,OAAO;IACT;IACA,IAAI,CAACF,eAAeqB,IAAI,CAAChB,OAAO,OAAO;IACvC,OAAOD,sBAAsBC,SAASF;AACxC;AAEA,uEAAuE,GACvE,OAAO,MAAMsB,qBAAqB,WAAU;AAE5C,kEAAkE,GAClE,MAAMC,wBAAwB,IAAIpC,IAAI;IACpC;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;CACD;AAED;;;;;;;;;;;;CAYC,GACD,OAAO,SAASqC,oBAAoBR,KAAa;IAC/C,IAAI,CAAC,gBAAgBE,IAAI,CAACF,QAAQ,OAAOA;IACzC,IAAIS;IACJ,IAAI;QACFA,MAAM,IAAIC,IAAIV;IAChB,EAAE,eAAM;QACN,OAAOA;IACT;IACA,IAAIW,UAAU;IACd,IAAIF,IAAIG,QAAQ,IAAIH,IAAII,QAAQ,EAAE;QAChCJ,IAAIG,QAAQ,GAAG;QACfH,IAAII,QAAQ,GAAG;QACfF,UAAU;IACZ;IACA,KAAK,MAAMG,QAAQC,MAAMC,IAAI,CAACP,IAAIQ,YAAY,CAACC,IAAI,IAAK;QACtD,IAAI,CAACX,sBAAsBY,GAAG,CAACL,KAAK/C,WAAW,KAAK;QACpD,IAAI,CAAC0C,IAAIQ,YAAY,CAAC1B,GAAG,CAACuB,OAAO;QACjCL,IAAIQ,YAAY,CAAC3B,GAAG,CAACwB,MAAMR;QAC3BK,UAAU;IACZ;IACA,OAAOA,UAAUF,IAAIW,QAAQ,KAAKpB;AACpC;AAEA;;;;;CAKC,GACD,OAAO,SAASqB,aAAazD,GAAY,EAAEoC,KAAe;IACxD,MAAMsB,SAAS3D,gBAAgBC;IAC/B,IAAI,CAAC0D,OAAO1B,MAAM,EAAE,OAAO;IAC3B,MAAM2B,OAAOD,OAAOhE,MAAM,CAAC,CAACkE,QAAUtD,aAAaiD,GAAG,CAACK;IACvD,IAAK,IAAIC,QAAQ,GAAGA,QAAQ,IAAIH,OAAO1B,MAAM,EAAE6B,SAAS,EAAG;QACzD,IAAIrD,kBAAkB+C,GAAG,CAAC,GAAGG,MAAM,CAACG,MAAM,CAAC,CAAC,EAAEH,MAAM,CAACG,QAAQ,EAAE,EAAE,GAAG;YAClE,OAAO;QACT;IACF;IACA,IAAI,CAACF,KAAK3B,MAAM,EAAE,OAAO;IACzB,wEAAwE;IACxE,MAAM8B,WAAWH,KAAKI,KAAK,CAAC,CAACH,QAAUnD,eAAe8C,GAAG,CAACK;IAC1D,IAAIE,YAAa,CAAA,OAAO1B,UAAU,YAAY,OAAOA,UAAU,SAAQ,GAAI;QACzE,OAAO;IACT;IACA,OAAO;AACT;AAEA,4EAA4E,GAC5E,SAAS4B,2BAA2BhE,GAAY;IAC9C,OAAOD,gBAAgBC,KAAKuC,IAAI,CAAC,CAACqB,QAAUlD,iBAAiB6C,GAAG,CAACK;AACnE;AAUA,MAAMK,iBAAiB,CAACC,UAAqC,CAAA;QAC3DC,UAAU;QACVD;QACAE,QAAQ;IACV,CAAA;AAEA;;;;;;;;;;;;;;;;;;CAkBC,GACD,OAAO,SAASC,cAAcjC,KAAc,EAAEpC,MAAM,EAAE;IACpD,oDAAoD;IACpD,IAAIA,OAAOyD,aAAazD,KAAKoC,QAAQ;QACnC,OAAO6B,eAAe7B,UAAUkC,aAAalC,UAAU,QAAQA,UAAU;IAC3E;IACA,IAAIA,UAAU,QAAQA,UAAUkC,WAAW,OAAOlC,gBAAAA,QAAS;IAC3D,IAAI,OAAOA,UAAU,UAAU;QAC7B,yEAAyE;QACzE,0DAA0D;QAC1D,MAAMmC,WAAW3B,oBAAoBR;QACrC,IAAImC,aAAanC,OAAO,OAAOmC;QAC/B,wEAAwE;QACxE,oCAAoC;QACpC,IAAIpC,oBAAoBC,UAAU,CAAC4B,2BAA2BhE,MAAM;YAClE,OAAOiE,eAAe;QACxB;QACA,OAAO7B;IACT;IACA,IAAIe,MAAMqB,OAAO,CAACpC,QAAQ,OAAOA,MAAMvC,GAAG,CAAC,CAAC4E,QAAUJ,cAAcI,OAAOzE;IAC3E,IAAI,OAAOoC,UAAU,UAAU;QAC7B,MAAMtC,SAASsC;QACf,qEAAqE;QACrE,uCAAuC;QACvC,IAAI,OAAO,AAACtC,OAAe4E,MAAM,KAAK,YAAY;YAChD,IAAI;gBACF,OAAO,AAAC5E,OAAe4E,MAAM,GAAGC,WAAW;YAC7C,EAAE,eAAM;gBACN,OAAO1E,OAAOmC;YAChB;QACF;QACA,IAAI,OAAO,AAACtC,OAAe8E,QAAQ,KAAK,YAAY,OAAO;QAC3D,MAAMC,MAA+B,CAAC;QACtC,KAAK,MAAM,CAACC,UAAUC,WAAW,IAAIC,OAAOC,OAAO,CAACnF,QAAS;YAC3D+E,GAAG,CAACC,SAAS,GAAGT,cAAcU,YAAYD;QAC5C;QACA,OAAOD;IACT;IACA,OAAOzC;AACT;AA2BA,8FAA8F,GAC9F,MAAM8C,YAAY;IAChB;IACA;IACA;IACA;CACD;AAED,SAASC,WAAWC,OAAsB;IACxC,OAAOjG,sBAAsBO,MAAM,CAAC,CAACI,SAAWA,OAAOR,QAAQ,CAAC+F,QAAQ,CAACD;AAC3E;AAEA,4EAA4E,GAC5E,eAAeE,YACbC,EAAO,EACPnG,UAAkB,EAClBoG,KAAa,EACbC,EAAU,EACVC,UAAoC,CAAC,CAAC;IAEtC,MAAMC,OAAO,MAAMJ,GAAGnG,UAAU,CAACA,YAAYwG,KAAK,CAACJ,OAAO,MAAMC,IAAI9D,GAAG;IACvE,OAAOgE,KAAKE,IAAI,CAAChG,GAAG,CAAC,CAACiG,MAAc,aAG9BJ,QAAQK,UAAU,KAAK,QAAQ,CAAC,IAAI;YAAEN,IAAIK,IAAIL,EAAE;QAAC;YACrDO,MAAM3B,cAAcyB,IAAIE,IAAI;;AAEhC;AAEA,gEAAgE,GAChE,eAAeC,QACbV,EAAO,EACPnG,UAAkB,EAClBqG,EAAU;IAEV,MAAMS,WAAW,MAAMX,GAAGnG,UAAU,CAACA,YAAY0G,GAAG,CAACL,IAAI9D,GAAG;IAC5D,IAAI,CAACuE,SAASC,MAAM,EAAE,OAAO,EAAE;IAC/B,OAAO;QAAC;YAAEV;YAAIO,MAAM3B,cAAc6B,SAASF,IAAI;QAAI;KAAE;AACvD;AAEA;;;;;;;;CAQC,GACD,eAAeI,YACbb,EAAO,EACPc,GAAQ,EACRC,QAAQ,CAAC,EACTC,WAAW,CAAC;QAQM;IANlB,MAAML,WAAW,MAAMG,IAAI1E,GAAG;IAC9B,MAAM6E,QAA4B,EAAE;IACpC,IAAIN,SAASC,MAAM,EAAE;QACnBK,MAAMC,IAAI,CAAC;YAAEC,MAAML,IAAIK,IAAI;YAAEV,MAAM3B,cAAc6B,SAASF,IAAI;QAAI;IACpE;IACA,IAAIM,SAASC,UAAU,OAAOC;IAC9B,MAAMG,YAAY,OAAA,OAAMN,IAAIO,eAAe,oBAAnBP,IAAIO,eAAe,MAAnBP,iBAAN,OAAkC,EAAE;IACtD,KAAK,MAAMQ,SAASF,SAAU;QAC5B,MAAMhB,OAAO,MAAMkB,MAAMlF,GAAG;QAC5B,KAAK,MAAMmE,OAAOH,KAAKE,IAAI,CAAE;YAC3BW,MAAMC,IAAI,IAAK,MAAML,YAAYb,IAAIO,IAAIO,GAAG,EAAEC,QAAQ,GAAGC;QAC3D;IACF;IACA,OAAOC;AACT;AASA;;;;;;;;CAQC,GACD,OAAO,eAAeM,eACpBC,GAAW,EACXrB,UAAyB,CAAC,CAAC;QAEhBA,oBASTA;IATF,MAAMH,MAAKG,qBAAAA,QAAQxG,SAAS,YAAjBwG,qBAAqB1G;IAChC,MAAMgH,OAA2C,CAAC;IAClD,MAAMzE,SAAiC,CAAC;IAExC,qEAAqE;IACrEyE,IAAI,CAAC,QAAQ,GAAG,MAAMI,YACpBb,IACAA,GAAGnG,UAAU,CAAC,SAAS0G,GAAG,CAACiB,MAC3B,IACArB,oBAAAA,QAAQa,QAAQ,YAAhBb,oBAAoB;IAGtB,sEAAsE;IACtEM,IAAI,CAAC,WAAW,GAAG,MAAMC,QAAQV,IAAI,YAAYwB;IAEjD,yEAAyE;IACzE,0EAA0E;IAC1E,0EAA0E;IAC1E,MAAMC,WAAW,MAAMzB,GACpB0B,eAAe,CAAC,YAChBrB,KAAK,CAAC,YAAY,MAAMmB,KACxBpF,GAAG,GACHuF,KAAK,CAAC,IAAO,CAAA;YAAErB,MAAM,EAAE;QAAU,CAAA;IACpCG,IAAI,CAAC,iBAAiB,GAAGgB,SAASnB,IAAI,CAAChG,GAAG,CAAC,CAACiG,MAAc,CAAA;YACxDY,MAAMZ,IAAIO,GAAG,CAACK,IAAI;YAClBV,MAAM3B,cAAcyB,IAAIE,IAAI;QAC9B,CAAA;IAEA,OAAO;QACLZ,SAAS;YAAE+B,MAAM;YAAQ1B,IAAIsB;QAAI;QACjCK,aAAa,IAAIC,OAAO1C,WAAW;QACnC2C,UAAUnC,WAAW;QACrBa;QACAzE;QACAgG,WAAWrC;IACb;AACF;AAEA;;;;;;;;;;CAUC,GACD,OAAO,eAAesC,cACpBC,KAAa,EACb/B,UAAyB,CAAC,CAAC;QAEhBA,oBAQTA;IARF,MAAMH,MAAKG,qBAAAA,QAAQxG,SAAS,YAAjBwG,qBAAqB1G;IAChC,MAAMgH,OAA2C,CAAC;IAClD,MAAMzE,SAAiC,CAAC;IAExCyE,IAAI,CAAC,OAAO,GAAG,MAAMI,YACnBb,IACAA,GAAGnG,UAAU,CAAC,QAAQ0G,GAAG,CAAC2B,QAC1B,IACA/B,oBAAAA,QAAQa,QAAQ,YAAhBb,oBAAoB;IAGtB,wDAAwD;IACxD,MAAMgC,QAAQ,MAAMnC,GAAGnG,UAAU,CAAC,SAASwG,KAAK,CAAC,SAAS,MAAM6B,OAAO9F,GAAG;IAC1E,MAAMgG,WAA+B,EAAE;IACvC,MAAMC,UAA8B,EAAE;IACtC,MAAMC,SAA6B,EAAE;IACrC,KAAK,MAAMC,QAAQJ,MAAM7B,IAAI,CAAE;YACwBH;QAArDiC,SAASlB,IAAI,IAAK,MAAML,YAAYb,IAAIuC,KAAKzB,GAAG,EAAE,IAAGX,qBAAAA,QAAQa,QAAQ,YAAhBb,qBAAoB;QACzEkC,QAAQnB,IAAI,IAAK,MAAMR,QAAQV,IAAI,aAAauC,KAAKrC,EAAE;QACvD,wEAAwE;QACxE,wEAAwE;QACxE,8DAA8D;QAC9DoC,OAAOpB,IAAI,IAAK,MAAMnB,YAAYC,IAAI,sBAAsB,UAAUuC,KAAKrC,EAAE;IAC/E;IACAO,IAAI,CAAC,QAAQ,GAAG2B;IAChB3B,IAAI,CAAC,YAAY,GAAG4B;IACpB5B,IAAI,CAAC,qBAAqB,GAAG6B;IAE7B,wEAAwE;IACxE,wDAAwD;IACxD,KAAK,MAAMzI,cAAc;QACvB;QACAL;QACA;QACA;QACA;QACA;QACA;KACD,CAAE;QACDiH,IAAI,CAAC5G,WAAW,GAAG,MAAMkG,YAAYC,IAAInG,YAAY,SAASqI;IAChE;IAEA,wEAAwE;IACxE,mBAAmB;IACnBzB,IAAI,CAAC,UAAU,GAAG,MAAMV,YAAYC,IAAI,WAAW,SAASkC,OAAO;QACjE1B,YAAY;IACd;IAEA,sEAAsE;IACtE,4EAA4E;IAC5E,sDAAsD;IACtDC,IAAI,CAAC,6BAA6B,GAAG,MAAMV,YACzCC,IACA,8BACA,SACAkC;IAGF,6EAA6E;IAC7EzB,IAAI,CAAC,gBAAgB,GAAG,MAAMV,YAAYC,IAAI,iBAAiB,SAASkC;IAExE,uEAAuE;IACvE,8DAA8D;IAC9D,sEAAsE;IACtE,8CAA8C;IAC9C,KAAK,MAAMrI,cAAc;QACvB;QACA;QACA;QACA;QACA;QACA;QACA;QACA;QACA;QACA;QACA;QACA;QACA;QACA;QACA;QACA;QACA;QACA;QACA;QACA;QACA;QACA;QACA;QACA;QACA;KACD,CAAE;QACD4G,IAAI,CAAC5G,WAAW,GAAG,MAAMkG,YAAYC,IAAInG,YAAY,SAASqI;IAChE;IAEAzB,IAAI,CAAC,oBAAoB,GAAG,MAAMC,QAAQV,IAAI,qBAAqBkC;IACnEzB,IAAI,CAAC,sBAAsB,GAAG,MAAMV,YAClCC,IACA,uBACA,aACAkC;IAGF,2EAA2E;IAC3E,2DAA2D;IAC3D,MAAMM,UAAU,MAAMxC,GACnBnG,UAAU,CAAC,kBACXwG,KAAK,CAAC,SAAS,MAAM6B,OACrB9F,GAAG;IACN,MAAMqG,aAAiC,EAAE;IACzC,KAAK,MAAMC,UAAUF,QAAQlC,IAAI,CAAE;QACjCmC,WAAWvB,IAAI,IAAK,MAAML,YAAYb,IAAI0C,OAAO5B,GAAG,EAAE,GAAG;IAC3D;IACAL,IAAI,CAAC,iBAAiB,GAAGgC;IAEzB,yDAAyD;IACzD,MAAME,aAAa,MAAM3C,GACtBnG,UAAU,CAAC,kBACXwG,KAAK,CAAC,SAAS,MAAM6B,OACrB9F,GAAG;IACNJ,MAAM,CAAC,iBAAiB,GAAG2G,WAAWrC,IAAI,CAAC7D,MAAM;IAEjD,OAAO;QACLoD,SAAS;YAAE+B,MAAM;YAAO1B,IAAIgC;QAAM;QAClCL,aAAa,IAAIC,OAAO1C,WAAW;QACnC2C,UAAUnC,WAAW;QACrBa;QACAzE;QACAgG,WAAWrC;IACb;AACF;AAEA,2EAA2E,GAC3E,OAAO,SAASiD,eAAe5I,QAA4B;IACzD,MAAM6I,MAAM7I,SAAS6H,WAAW,CAACiB,KAAK,CAAC,GAAG;IAC1C,OAAO,CAAC,MAAM,EAAE9I,SAAS6F,OAAO,CAAC+B,IAAI,CAAC,MAAM,EAAE5H,SAAS6F,OAAO,CAACK,EAAE,CAAC,CAAC,EAAE2C,IAAI,KAAK,CAAC;AACjF;AAEA,eAAetB,eAAc"}
|
|
1
|
+
{"version":3,"sources":["../../../../../../../../libs/tenant/data/admin/src/lib/server/personal-data-export.ts"],"sourcesContent":["/**\n * @license\n * Copyright 2026 Aglyn LLC\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n/**\n * Personal-data export — access and portability, in a machine-readable form\n * (AGL-1974).\n *\n * Privacy Policy §7 grants portability (GDPR Art. 20) and nothing in the\n * product exported a person's or a workspace's personal data. Six surfaces\n * told customers to \"export first\" before deleting — the org settings Delete\n * tab, the Close account card, `erase-org-cli.mjs`, the staff org actions,\n * live DPA §11 and Terms §13.3 — all pointing at something that did not\n * exist. An access request was assembled by a staff member reading the console\n * by hand.\n *\n * ## THE PROPERTY THIS FILE EXISTS FOR\n *\n * **An export whose coverage differs from erasure's is a bug in one of them.**\n * Both answer the same question — *what do we hold about this subject* — and\n * the honest failure modes are symmetric: an export that misses a collection\n * under-discloses on an access request, and an erasure that misses the same\n * one leaves data behind after we said it was gone. Historically the second\n * kept happening (AGL-1444, AGL-1448, AGL-1970, AGL-1971 each found a\n * collection outside the cascade), which is exactly the evidence that a list\n * kept in one place and mirrored by hand in another will drift.\n *\n * The ideal is one enumeration both derive from. That is not available here:\n * `erase.ts` is under concurrent change and its sweep list is expressed as\n * code, not data. So the coverage is enumerated ONCE below, as data, and\n * `personal-data-export-coverage.spec.ts` DISCOVERS every collection\n * `erase.ts` touches out of its source and fails when the two disagree. A new\n * sweep added to the erasure turns this file red until somebody decides\n * whether the export discloses it — which is the decision that was being\n * skipped.\n *\n * That guard is worth more than the feature.\n *\n * ## What must NOT come out wholesale\n *\n * Three categories, and each is a way an access request becomes a breach:\n *\n * 1. **Another person's data.** A workspace is shared. A member asking what\n * we hold about THEM gets their own roster row, not the roster; their own\n * support messages, not the thread's other authors. Handled by exporting\n * the person and the workspace as different subjects with different\n * scopes, never by filtering a single blob after the fact.\n * 2. **Secrets.** An `apiKeys` row confirms that a credential exists — its\n * label, scopes, creator and dates, which are all facts about the subject\n * — and never re-discloses the credential. Note the document ID *is* the\n * SHA-256 of the token, so the id itself is withheld too; a hash handed\n * back is a verifier handed back.\n * 3. **Staff-only fields.** Internal notes, override reasons and moderation\n * state are about our handling, not about the person, and several name a\n * staff member.\n *\n * ## What it is NOT\n *\n * Not `GET /api/hosts/export` — that is a Pro+ site-design backup whose own\n * header says it never includes bookings, leads, submissions or secrets, and\n * offering it as a portability answer would be misdescribing it. Not the\n * `firestore-export` cron, which is an unscoped DR snapshot. Not the audit CSV.\n */\n\nimport { firebaseAdmin } from './firebase-admin'\nimport { CONSOLE_DOMAINS_COLLECTION } from './console-domains'\n\nconst firestoreApp = () => firebaseAdmin.app().firestore()\n\n/** Which subject an export source belongs to. */\nexport type ExportSubject = 'user' | 'org'\n\n/** How a source is located — the axis a path cascade is blind to. */\nexport type ExportKeying =\n /** Top-level collection carrying the subject id in a FIELD. */\n | 'field'\n /** Top-level collection whose DOCUMENT ID is the subject id. */\n | 'doc-id'\n /** Under the subject's own document path. */\n | 'path'\n /** Reached through another collection's rows (a collection group). */\n | 'collection-group'\n /** Outside Firestore entirely. */\n | 'external'\n\nexport interface ExportSourceSpec {\n /** Firestore collection name, or an `external:` pseudo-name. */\n collection: string\n keyedBy: ExportKeying\n subjects: readonly ExportSubject[]\n /**\n * `true` when the export discloses it. `false` requires `reason` — and the\n * reason is the whole value of the field: it is a decision recorded at the\n * moment it was made, not an omission discovered later.\n */\n exported: boolean\n /** Why it is withheld, or how it is disclosed. Always present. */\n note: string\n}\n\n/**\n * Every collection `erase.ts` touches, and what the export does with it.\n *\n * Kept in the SAME order as the erasure's steps so a reader comparing the two\n * files reads them in step. `personal-data-export-coverage.spec.ts` proves the\n * set matches; this list carries the decision for each one.\n */\nexport const PERSONAL_DATA_SOURCES: readonly ExportSourceSpec[] = [\n // ── The person ────────────────────────────────────────────────────────\n {\n collection: 'users',\n keyedBy: 'doc-id',\n subjects: ['user'],\n exported: true,\n note: 'Profile: name, phone, postal address, legalAcceptances, and every subcollection (orgs, hostMemberships, notifications, passkeys).',\n },\n {\n collection: 'profiles',\n keyedBy: 'doc-id',\n subjects: ['user'],\n exported: true,\n note: 'The world-readable handle, display name and Stripe Connect account. Not reachable from the user tree (AGL-1970) — the same blind spot on both sides.',\n },\n // ── The workspace ─────────────────────────────────────────────────────\n {\n collection: 'orgs',\n keyedBy: 'doc-id',\n subjects: ['org'],\n exported: true,\n note: 'The org document and every subcollection: members, invites, roles, usage, apiUsage, analytics, datasets/records, contacts, lists, media, installs, activity, assistExchanges, retention.',\n },\n {\n collection: 'hosts',\n keyedBy: 'field',\n subjects: ['org'],\n exported: true,\n note: 'Each site tree the org owns — pages, layouts, orders, form submissions. Secrets within are redacted, not omitted, so the export still says a webhook exists.',\n },\n {\n collection: 'hostIndex',\n keyedBy: 'doc-id',\n subjects: ['org'],\n exported: true,\n note: 'Subdomain/CNAME routing for each of the org’s sites. Configuration the customer set, so it is theirs to take.',\n },\n {\n collection: 'supplierDeliveries',\n keyedBy: 'field',\n subjects: ['org'],\n exported: true,\n note: 'The dropship outbox for each of the org’s sites, keyed by hostId (AGL-1448). Disclosed to the ORG and to no one else, and the reason is the unusual one: the personal data in the row is a SHOPPER’s — name, email and shipping address in the body we sign and POST to the supplier. That is the merchant’s own order record, which they already read in their console and for which they are the controller and we the processor, so an org access request is entitled to it. It is NOT a `user`-subject source: a console member asking what we hold about THEM is not the shopper, and returning these rows would answer their request with a third party’s address. Withholding it was not an option worth taking — `eraseHost` destroys these rows, so an export that omitted them would delete what it never disclosed.',\n },\n // ── Top-level, field-keyed: what a path cascade cannot see ────────────\n {\n collection: 'apiKeys',\n keyedBy: 'field',\n subjects: ['org'],\n exported: true,\n note: 'EXISTENCE ONLY — label, scopes, creating uid, dates. Never the credential, and never the document id, which is the SHA-256 of the token.',\n },\n {\n collection: 'outreachMailboxCredentials',\n keyedBy: 'field',\n subjects: ['org'],\n exported: true,\n note: 'EXISTENCE ONLY — which connected mailbox a credential authorizes, its provider, granted scopes and dates (AGL-2974), and since AGL-2978 the connected account’s address, its Google account id and the member who connected it: the organization’s own record of whose mailbox it holds. Every token is redacted by name and by shape (see redactSecrets) and never disclosed; the sealed refresh token and the id of the key that sealed it both carry `token` in their names. The document id is the mailbox id, not derived from any secret, so it is disclosed.',\n },\n {\n collection: 'outreachLinks',\n keyedBy: 'field',\n subjects: ['org'],\n exported: true,\n note: 'Where each short tracking link in a sent Outreach sequence email points (AGL-3297): the enrollment, the step, the link’s index and the destination the step wrote. The organization’s own record of what it sent; the id is random and derived from no secret.',\n },\n {\n collection: 'shippingTrackers',\n keyedBy: 'field',\n subjects: ['org'],\n exported: true,\n note: 'One row per tracking number a label was bought for (AGL-3612): the site, the order record, the carrier, the tracking number and its latest status. The organization’s own record of what it shipped; the order it points at is already disclosed under its site. Org-only, for the reason `supplierDeliveries` gives: a tracking number locates a shopper’s parcel.',\n },\n {\n collection: 'shippingQuoteCache',\n keyedBy: 'field',\n subjects: ['org'],\n exported: true,\n note: 'Carrier quotes for a site’s checkout, kept ten minutes (AGL-3612): the site, each service’s price and transit estimate, and when the row expires. No address is stored; the document id is a SHA-256 of the quoted request, an identifier rather than a verifier of any credential. Disclosed while it exists so the export does not omit a collection the erasure destroys.',\n },\n {\n collection: 'taxEngineConnections',\n keyedBy: 'field',\n subjects: ['org'],\n exported: true,\n note: 'EXISTENCE ONLY for the credential — each site’s connection to the merchant’s own Avalara AvaTax or TaxJar account (AGL-3631): provider, environment, account id and company code, ship-from address, default tax code, last test result and the member who connected it. The sealed API token and the id of the key that sealed it both carry `token` in their names and are redacted (see redactSecrets); the document id is the site id.',\n },\n {\n collection: 'taxEngineProductCodes',\n keyedBy: 'field',\n subjects: ['org'],\n exported: true,\n note: 'The tax code the merchant assigned each product at their tax service (AGL-3631). Catalog configuration the customer set, so it is theirs to take.',\n },\n {\n collection: 'taxEngineExemptions',\n keyedBy: 'field',\n subjects: ['org'],\n exported: true,\n note: 'Customers the merchant recorded as tax-exempt (AGL-3631): email, name, exemption type, certificate number and the regions it covers. A SHOPPER’s data held for the merchant as controller, so disclosed to the ORG only, for the reason `supplierDeliveries` gives. The document id carries a truncated SHA-256 of the email — a lookup key, not a credential.',\n },\n {\n collection: 'taxEngineTransactions',\n keyedBy: 'field',\n subjects: ['org'],\n exported: true,\n note: 'Each paid order as recorded with the merchant’s tax service, and the refunds reversed against it (AGL-3631): the sale sent (lines, ship-to address, the customer code, which is the shopper’s email), its status and last error. The merchant’s own tax record and a shopper’s address, so disclosed to the ORG only.',\n },\n {\n collection: 'marketingPlatformConnections',\n keyedBy: 'field',\n subjects: ['org'],\n exported: true,\n note: 'EXISTENCE ONLY for the credential — each site’s connection to the merchant’s own Mailchimp, Klaviyo, Omnisend or Attentive account (AGL-3639): provider, status, account name, the chosen list and tag, the sync switches, cursors and run timing. The sealed access and refresh tokens and the id of the key that sealed them carry `token` in their names and are redacted (see redactSecrets); the document id is the site id and the provider.',\n },\n {\n collection: 'marketingPlatformEvents',\n keyedBy: 'field',\n subjects: ['org'],\n exported: true,\n note: 'Commerce events still owed to a connected marketing platform (AGL-3639): checkout started, order paid, fulfilled, refunded or cancelled, with the shopper’s email, the order, its items and value, and the tracking number. A SHOPPER’s data held for the merchant as controller, so disclosed to the ORG only, for the reason `supplierDeliveries` gives; a row is deleted once delivered.',\n },\n {\n collection: 'helpDeskConnections',\n keyedBy: 'field',\n subjects: ['org'],\n exported: true,\n note: 'EXISTENCE ONLY for the credential — each site’s connection to the merchant’s own Gorgias, Re:amaze, Kustomer, Gladly or Richpanel account (AGL-3824): provider, status, the account’s subdomain and name, the Richpanel support address, the chosen channel, inbox or custom object, the sync switches, cursors and run timing. The sealed API key, login and app client id, sealed together, and the id of the key that sealed them carry `token` in their names and are redacted (see redactSecrets); the document id is the site id and the provider.',\n },\n {\n collection: 'helpDeskOutbox',\n keyedBy: 'field',\n subjects: ['org'],\n exported: true,\n note: 'Orders and contact form messages still owed to a connected help desk (AGL-3824): a buyer’s email and name with the order’s number, items and totals, or a message’s sender, subject and text. A SHOPPER’s or a visitor’s data held for the merchant as controller, so disclosed to the ORG only, for the reason `supplierDeliveries` gives; a row is deleted once delivered.',\n },\n {\n collection: 'helpDeskCustomers',\n keyedBy: 'field',\n subjects: ['org'],\n exported: true,\n note: 'What each help desk connection knows of a buyer (AGL-3824): their email and name, the help desk’s id for them and their ten latest orders, so each order sent carries the recent ones. A SHOPPER’s data held for the merchant as controller, so disclosed to the ORG only, for the reason `supplierDeliveries` gives. The document id carries a truncated SHA-256 of the email — a lookup key, not a credential.',\n },\n {\n collection: 'adConversionConnections',\n keyedBy: 'field',\n subjects: ['org'],\n exported: true,\n note: 'EXISTENCE ONLY for the credential — each site’s Conversions API connection to the merchant’s own Meta, TikTok or Pinterest ad account (AGL-3694): provider, status, the Pinterest ad account id, the test event code, and what was last sent or refused. The sealed access token and the id of the key that sealed it carry `token` in their names and are redacted (see redactSecrets); the document id is the site id and the provider.',\n },\n {\n collection: 'adConversionConsents',\n keyedBy: 'field',\n subjects: ['org'],\n exported: true,\n note: 'The advertising consent a shopper’s checkout carried, recorded only where it granted advertising on a site with a connection (AGL-3694): the consent status and when it was decided, the country, the page, the ad vendors’ browser ids, the IP address and the user agent. A SHOPPER’s data held for the merchant as controller, so disclosed to the ORG only; deleted when the order is reported or after 14 days.',\n },\n {\n collection: 'adConversionEvents',\n keyedBy: 'field',\n subjects: ['org'],\n exported: true,\n note: 'Purchases and leads owed to a connected ad account (AGL-3694): the event id, order id, value and items, the visitor’s details as SHA-256 hashes, and the IP address, user agent and vendor browser ids. A SHOPPER’s data held for the merchant as controller, so disclosed to the ORG only; the personal fields are dropped once the event is sent, and every row expires after 14 days.',\n },\n {\n collection: 'fulfillmentNetworkConnections',\n keyedBy: 'field',\n subjects: ['org'],\n exported: true,\n note: 'EXISTENCE ONLY for the grant — each site’s connection to the merchant’s own ShipBob, ShipMonk, Flexport or Amazon Multi-Channel Fulfillment account (AGL-3634, AGL-3697, AGL-3822): network, status, account name, ShipBob channel, ShipMonk store id, Amazon marketplaces, routing and stock settings, the last stock count by SKU and totals. The sealed access and refresh tokens (for ShipMonk and Flexport, the merchant’s own API key or token, sealed as the access token), the webhook token’s hash and the id of the key that sealed them carry `token` in their names, and ShipMonk’s sealed webhook signing secret carries `secret` in its name; all are redacted (see redactSecrets). The document id is the site id and the network.',\n },\n {\n collection: 'fulfillmentNetworkOrders',\n keyedBy: 'field',\n subjects: ['org'],\n exported: true,\n note: 'Each paid order’s hand-off to a fulfillment network (AGL-3634): the order number, the lines sent with SKU and quantity, the network’s reference, the parcels read back with carrier, tracking number and status, and why anything was not sent. The merchant’s own fulfillment record of a SHOPPER’s order, so disclosed to the ORG only.',\n },\n {\n collection: 'courierConnections',\n keyedBy: 'field',\n subjects: ['org'],\n exported: true,\n note: 'EXISTENCE ONLY for the keys — each site’s connection to the merchant’s own DoorDash Drive developer account (AGL-3695): provider, the developer and key ids of its live and test keys, each key’s last test result, the store’s courier phone and note, and the member who connected it. The sealed signing secrets carry `secret` and the webhook token’s hash carries `token` in their names and are redacted (see redactSecrets); the document id is the site id and the provider.',\n },\n {\n collection: 'courierDeliveries',\n keyedBy: 'field',\n subjects: ['org'],\n exported: true,\n note: 'Each local-delivery order’s courier (AGL-3695): the quote offered, the run booked and the runs before it — the courier’s reference, status, fee, arrival estimates, tracking link and any cancellation reason, and the member who booked it. No address or phone is stored here (they stay on the order); the merchant’s own record of a SHOPPER’s delivery, so disclosed to the ORG only.',\n },\n {\n collection: 'marketplaceConnections',\n keyedBy: 'field',\n subjects: ['org'],\n exported: true,\n note: 'EXISTENCE ONLY for the grant — each site’s connection to the merchant’s own Amazon, eBay, Etsy, TikTok Shop, Walmart or Faire seller account (AGL-3638): marketplace, status, account name, the marketplace sites the account sells on, the listing and order settings, the order cursor and the last sync’s counts. The sealed access and refresh tokens and the id of the key that sealed them carry `token` in their names and are redacted (see redactSecrets); the account’s marketplace ids are disclosed as stored. The document id is the site id and the marketplace. Its `log` and `listingState` subcollections (what each listing was last sent) are not read by this export.',\n },\n {\n collection: 'marketplaceOrders',\n keyedBy: 'field',\n subjects: ['org'],\n exported: true,\n note: 'Each marketplace order the store imported (AGL-3638): the marketplace and its order id, the store order it became, which order line each marketplace line became, the marketplace’s fees, and each shipment confirmed back with carrier and tracking number. The merchant’s own record of a marketplace BUYER’s order, so disclosed to the ORG only; the buyer’s name and address are on the store’s order, not here.',\n },\n {\n collection: 'inventorySyncConnections',\n keyedBy: 'field',\n subjects: ['org'],\n exported: true,\n note: 'EXISTENCE ONLY for the keys — each site’s connection to the merchant’s own Cin7 Core, inFlow, Brightpearl or Ordoro account (AGL-3642): system, status, account name, the stock, product and order settings (location, the customer or Ordoro sales channel orders are recorded under, the tax rule), the last count by SKU, the last syncs’ counts, cursors and totals. The sealed API keys or grant and the id of the key that sealed them carry `credential` in their names and are redacted (see redactSecrets); Brightpearl’s account code and datacenter are disclosed as stored. The document id is the site id. Its `log` subcollection is not read by this export.',\n },\n {\n collection: 'inventorySyncOrders',\n keyedBy: 'field',\n subjects: ['org'],\n exported: true,\n note: 'Each paid order’s hand-off to the connected inventory system (AGL-3642): the order number and our reference, the lines sent with SKU, quantity and price, the buyer’s name, email address and shipping address as sent, the totals, the system’s id and number for it, attempts and why it was not sent. The merchant’s own record of a SHOPPER’s order, so disclosed to the ORG only.',\n },\n {\n collection: 'inventorySyncProducts',\n keyedBy: 'field',\n subjects: ['org'],\n exported: true,\n note: 'Each product linked between the store and the connected inventory system (AGL-3642): the system’s product id, the SKU, the version last synced, the store’s product and variant ids, and why it was not synced. Catalog data, no person’s.',\n },\n {\n collection: 'podConnections',\n keyedBy: 'field',\n subjects: ['org'],\n exported: true,\n note: 'EXISTENCE ONLY for the tokens — each site’s connection to the merchant’s own Printful store or Printify shop (AGL-3641): service, store id and name, currency, whether orders are sent automatically or held for review, whether prices sync, how shipments are heard about, and the last error. The sealed API token and webhook token and the id of the key that sealed them carry `token` in their names and are redacted (see redactSecrets).',\n },\n {\n collection: 'podProductLinks',\n keyedBy: 'field',\n subjects: ['org'],\n exported: true,\n note: 'Each product imported from a print-on-demand service (AGL-3641): the service’s product and variant ids, the store product and variants it became, names, SKUs, what the service charges for each variant and the store’s price. Catalog data, no person’s.',\n },\n {\n collection: 'podOrders',\n keyedBy: 'field',\n subjects: ['org'],\n exported: true,\n note: 'The part of each paid order a print-on-demand service makes and ships (AGL-3641): the store order and its number, the service’s order id, the lines sent with quantity and cost, its status, costs, attempts and errors, and each parcel’s carrier and tracking number. No buyer name, address or email is kept here; they are on the store’s order. The merchant’s own record of a SHOPPER’s order, so disclosed to the ORG only.',\n },\n {\n collection: 'deliveryAppStores',\n keyedBy: 'field',\n subjects: ['org'],\n exported: true,\n note: 'Each site’s link to the merchant’s own DoorDash, Uber Eats or Grubhub store (AGL-3644): the service, the store id it shows the merchant, the settings (accept automatically, prep minutes), the items matched to products and those waiting, and the menu’s last send. No credential is stored: every call is signed with the deployment’s partner credentials.',\n },\n {\n collection: 'deliveryAppOrders',\n keyedBy: 'field',\n subjects: ['org'],\n exported: true,\n note: 'Each order a delivery service sent a site’s register (AGL-3644): the service and its order id and short code, the items with their options and instructions, the totals and refunds, where it stood at the counter, and the store order it became. The buyer appears by first name and last initial only. The merchant’s own record of a delivery BUYER’s order, so disclosed to the ORG only.',\n },\n {\n collection: 'zapierHooks',\n keyedBy: 'field',\n subjects: ['org'],\n exported: true,\n note: 'Each Zapier REST hook a site’s API key subscribed (AGL-3643): the site, the events, the API key’s public id and name, delivery counts and dates. EXISTENCE ONLY for the URL — `targetUrl` is the capability Zapier minted for the Zap, a bearer URL whose secret is in its path, so it is redacted by name (see redactSecrets).',\n },\n {\n collection: 'zapierHookDeliveries',\n keyedBy: 'field',\n subjects: ['org'],\n exported: true,\n note: 'Markers that one event reached one Zapier hook, so a retry does not post it twice (AGL-3643): the hook, the status and the time. The document id is a SHA-256 of the event and hook, an identifier rather than a verifier of any credential; no customer data, and a TTL policy removes each row.',\n },\n {\n collection: 'ssoDomains',\n keyedBy: 'field',\n subjects: ['org'],\n exported: true,\n note: 'The public SSO routing document: domain, GCIP tenant and provider ids, IdP display name. `token` is redacted.',\n },\n {\n collection: CONSOLE_DOMAINS_COLLECTION,\n keyedBy: 'field',\n subjects: ['org'],\n exported: true,\n note: 'Custom console domain claims the org holds.',\n },\n {\n collection: 'apiIdempotency',\n keyedBy: 'field',\n subjects: ['org'],\n exported: false,\n note: 'Replay keys. The document id is a SHA-256 of the org id and the caller’s Idempotency-Key, and the row names a record id and nothing about a person. Disclosed as a COUNT so the export does not silently omit a collection the erasure destroys.',\n },\n {\n collection: 'stripeCustomers',\n keyedBy: 'field',\n subjects: ['org'],\n exported: true,\n note: 'The local reverse index: which Stripe customer id belongs to this workspace. An identifier the customer already holds on their own invoices.',\n },\n {\n collection: 'orgSlugs',\n keyedBy: 'field',\n subjects: ['org'],\n exported: true,\n note: 'Every workspace name the org has ever held, rename tombstones included — naming history is a fact about them.',\n },\n {\n collection: 'publisherHandles',\n keyedBy: 'field',\n subjects: ['org'],\n exported: true,\n note: 'Marketplace handle reservations and rename tombstones (AGL-1970).',\n },\n {\n collection: 'publisherProfiles',\n keyedBy: 'doc-id',\n subjects: ['org'],\n exported: true,\n note: 'The public publisher identity — handle, display name, bio, website, avatar, publisher agreement. `stripeAccountId` is the org’s own payout destination and is disclosed; nothing here is a credential.',\n },\n {\n collection: 'marketplaceListings',\n keyedBy: 'field',\n subjects: ['org'],\n exported: true,\n note: 'Listings the org published. These deliberately OUTLIVE an erasure (AGL-1448 Tier 3) — which makes disclosing them on an access request more important, not less: they are the part that does not go away.',\n },\n {\n collection: 'supportTickets',\n keyedBy: 'field',\n subjects: ['org', 'user'],\n exported: true,\n note: 'For an ORG: every thread and its messages. For a PERSON: only the messages they themselves authored — a thread has other authors, and handing a member the whole conversation would disclose colleagues’ words to them.',\n },\n // ── Retained past erasure on statutory grounds ────────────────────────\n {\n collection: 'platformRevenue',\n keyedBy: 'field',\n subjects: ['org'],\n exported: true,\n note: 'Per-transaction tax filing records. GDPR Art. 17(3)(b) exempts them from ERASURE (AGL-1811); it does not exempt them from ACCESS. If a transaction of theirs is in there it is disclosable — and the export is the only place a customer can see what survived their erasure.',\n },\n {\n collection: 'storefrontTaxCollected',\n keyedBy: 'field',\n subjects: ['org'],\n exported: true,\n note: 'Sales tax charged to shoppers on the org’s storefront (AGL-1904). Same statutory retention, same access obligation.',\n },\n // ── Records of our own handling ───────────────────────────────────────\n {\n collection: 'adminAudit',\n keyedBy: 'field',\n subjects: ['org', 'user'],\n exported: false,\n note: 'STAFF-ONLY. Rows name the staff actor, carry override reasons and internal notes, and are the record of our handling rather than data about the subject. `/admin/audit` → Export CSV serves a request that specifically asks for them, after a staff read.',\n },\n]\n\n/** Sources this export deliberately withholds, with the reason. */\nexport const EXPORT_WITHHELD = PERSONAL_DATA_SOURCES.filter((s) => !s.exported)\n\n/** Collection names the export claims to cover, for the coverage guard. */\nexport const EXPORT_COVERED_COLLECTIONS: readonly string[] =\n PERSONAL_DATA_SOURCES.map((source) => source.collection)\n\n/**\n * ## WHY THIS IS SHAPE-DRIVEN AND NOT A LIST OF NAMES\n *\n * It used to be a list of names, anchored on separators:\n *\n * /(^|[._-])(secret|token|password|apikey|…)($|[._-])/i\n *\n * That pattern requires a `.`, `_`, `-`, or a string boundary on both sides\n * of the word, so it reads `password_hash` and `webhooks.secret` and misses\n * `supplierToken` and `passwordScrypt` — the exact two fields the AGL-1881\n * review found leaving the platform in a customer's own export file. One is\n * a live bearer credential a supplier authenticates with; the other is a\n * shopper's scrypt password hash. **A denylist of names is always one naming\n * convention behind, and camelCase is the proof.**\n *\n * So the name half is now tokenized rather than separator-anchored, and it is\n * no longer the only gate. Three run, in order, and any of them redacts:\n *\n * * **(A) The name, tokenized.** `supplierToken` splits to\n * `['supplier','token']` the same way `supplier_token` and\n * `SUPPLIER-TOKEN` do, so the convention stops mattering. A name matches\n * on a single secret word or on an adjacent pair (`client secret`).\n * * **(B) The VALUE's shape, whatever the key is called.** This is the half\n * that does not need to know the schema: a PEM block, a JWT, a vendor\n * prefix (`sk_live_`, `whsec_`, `ghp_`, `AKIA`…), a `$2b$`/`$argon2` hash,\n * this repo's own `salt:hash` scrypt form, or an opaque high-entropy\n * string is a credential whether the field is called `supplierToken`,\n * `t`, or `notes`. **Gate B is the \"redact unless known safe\" default**:\n * a credential-shaped string is withheld unless its key names it as a\n * content identifier (see {@link SAFE_VALUE_WORDS}).\n * * **(C) Credentials carried inside a URL.** `?token=`, `?sig=`,\n * `?key=`, and `https://user:pass@host` are stripped from the URL rather\n * than the URL being blanked, because a media object's `url` field really\n * does carry a live Firebase download token in a query parameter, and the\n * location of the customer's own file is exactly what an access request\n * asks for. Precision beats blanking here.\n *\n * ### Why the KEY half is still deny-based, when the value half is not\n *\n * A true allowlist of field names is not available on this data. The export\n * walks documents whose field names our customers author — canvas nodes,\n * dataset records, custom metadata, form submissions — so an allowlist of\n * names would blank most of the file and turn an access request into a\n * non-answer, which is the under-disclosure this feature exists to stop. The\n * allowlist therefore lives where the vocabulary IS ours: value shapes. An\n * unrecognized VALUE shape is passed through (it is prose, a number, a date,\n * a customer's own content); an unrecognized value shape that looks like a\n * credential is withheld. That is the inversion the review asked for, put\n * where it can hold.\n *\n * ### The one deliberate carve-out\n *\n * `token`/`tokens` is also a metering word — `tokensUsed: 14203` on an\n * assist exchange is a usage count, and blanking it would blank a real\n * disclosure. A credential is a STRING; a count is a number. So a numeric or\n * boolean value under a name that matched only on a metering word passes\n * through. Nothing else in the vocabulary gets that treatment.\n *\n * A false positive costs a redacted field in an export. A false negative\n * costs a credential. Everything below is tuned in that direction.\n */\n\n/**\n * Split a field name into lowercase word tokens, convention-independently.\n *\n * `supplierToken`, `supplier_token`, `SUPPLIER-TOKEN` and `Supplier.Token`\n * all become `['supplier','token']`. Acronym runs split at the last capital\n * (`SSOTokenId` → `['sso','token','id']`) and digits break a word\n * (`apiKey2` → `['api','key','2']`), so a numbered field cannot hide.\n */\nexport function fieldNameTokens(key: unknown): string[] {\n return String(key ?? '')\n .replace(/[^A-Za-z0-9]+/g, ' ')\n .replace(/([a-z0-9])([A-Z])/g, '$1 $2')\n .replace(/([A-Z]+)([A-Z][a-z])/g, '$1 $2')\n .replace(/([A-Za-z])([0-9])/g, '$1 $2')\n .toLowerCase()\n .split(' ')\n .filter(Boolean)\n}\n\n/**\n * Words that make a field a credential on their own.\n *\n * Deliberately short. Every entry here is a word that has no innocent\n * meaning on a stored VALUE — `hmac`, `scrypt` and `bcrypt` name the\n * algorithm that produced a verifier, `passphrase` and `credential` name the\n * thing itself. `key` is NOT here and must never be: `publicKey` on a\n * passkey and `sortKey` on a dataset are not secrets, which is what\n * {@link SECRET_WORD_PAIRS} is for.\n */\nconst SECRET_WORDS = new Set([\n 'secret',\n 'secrets',\n 'password',\n 'passwords',\n 'passwd',\n 'pwd',\n 'passphrase',\n 'token',\n 'tokens',\n 'credential',\n 'credentials',\n 'apikey',\n 'privatekey',\n 'scrypt',\n 'bcrypt',\n 'argon',\n 'argon2',\n 'pbkdf2',\n 'hmac',\n 'otp',\n 'totp',\n 'authorization',\n])\n\n/**\n * Words that are a credential only next to another word.\n *\n * `key` is meaningless alone and decisive after `api`, `private`, `signing`,\n * `secret`, `session` or `encryption`; `token` after `access`/`refresh` is\n * already caught by {@link SECRET_WORDS} and listed here anyway so the pair\n * table reads as the complete statement of the rule.\n */\nconst SECRET_WORD_PAIRS = new Set([\n 'api key',\n 'private key',\n 'signing key',\n 'secret key',\n 'session key',\n 'encryption key',\n 'client secret',\n 'access token',\n 'refresh token',\n 'webhook secret',\n 'shared secret',\n // A Stripe payment link is a live bearer URL: whoever holds it can pay.\n // Named in the AGL-1443 inventory of what the old dump actually carried.\n 'payment link',\n // A Zapier REST hook's `targetUrl` (AGL-3643) is the same kind of bearer\n // URL: whoever holds it can post into the Zap, and its secret is in the\n // path, which the URL scrub does not read.\n 'target url',\n 'auth code',\n 'recovery code',\n 'backup code',\n])\n\n/**\n * The metering carve-out — see the header. A name that matched ONLY on one\n * of these, over a number or a boolean, is a count and is disclosed.\n */\nconst METERING_WORDS = new Set(['token', 'tokens'])\n\n/**\n * Key words that mark a high-entropy value as a content identifier rather\n * than a credential, defusing gate (B) only.\n *\n * A SHA-256 has exactly the shape of a 64-character secret and is not one.\n * Gate (A) is never defused by this list: `apiKeyId` is redacted, because\n * the name says credential and an id of a credential is not worth the\n * argument.\n */\nconst SAFE_VALUE_WORDS = new Set([\n // A passkey's `publicKey` is public by definition and has exactly the shape\n // of a secret. Safe to list because gate (A) is never defused here — a\n // `publicToken` is still redacted, on the strength of `token`.\n 'public',\n 'id',\n 'ids',\n 'uid',\n 'uuid',\n 'guid',\n 'hash',\n 'hashes',\n 'sha',\n 'sha1',\n 'sha256',\n 'sha512',\n 'md5',\n 'crc',\n 'crc32c',\n 'etag',\n 'checksum',\n 'digest',\n 'fingerprint',\n 'generation',\n 'version',\n 'revision',\n 'slug',\n 'path',\n 'url',\n 'uri',\n 'href',\n 'src',\n 'color',\n 'colour',\n 'hex',\n])\n\n/** Vendor credential prefixes — unambiguous, and worth naming explicitly. */\nconst CREDENTIAL_PREFIXES = [\n 'sk_live_',\n 'sk_test_',\n 'rk_live_',\n 'rk_test_',\n 'whsec_',\n 'ghp_',\n 'gho_',\n 'ghu_',\n 'ghs_',\n 'ghr_',\n 'github_pat_',\n 'xoxb-',\n 'xoxp-',\n 'xapp-',\n 'xoxa-',\n 'AKIA',\n 'ASIA',\n 'AIza',\n 'ya29.',\n 'SG.',\n 're_',\n 'shpat_',\n 'shpss_',\n 'npm_',\n 'glpat-',\n 'sq0csp-',\n 'sq0atp-',\n]\n\n/** A PEM block that carries a private key or a certificate request. */\nconst PEM_PATTERN = /-----BEGIN [A-Z0-9 ]*PRIVATE KEY-----/\n\n/** Modular crypt format: bcrypt, argon2, scrypt, PBKDF2, sha512-crypt. */\nconst CRYPT_HASH_PATTERN = /^\\$(2[abxy]?|argon2[a-z]*|scrypt|pbkdf2[^$]*|[56])\\$/\n\n/** This repo's own scrypt storage form: `{16+ hex salt}:{32+ hex hash}`. */\nconst SALTED_HASH_PATTERN = /^[0-9a-f]{16,}:[0-9a-f]{32,}$/i\n\n/** A JSON Web Token — three base64url segments. */\nconst JWT_PATTERN = /^[A-Za-z0-9_-]{8,}\\.[A-Za-z0-9_-]{8,}\\.[A-Za-z0-9_-]{8,}$/\n\n/** A UUID, which is an identifier far more often than it is a credential. */\nconst UUID_PATTERN =\n /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i\n\n/** Charset of an opaque credential: base64, base64url or hex, nothing else. */\nconst OPAQUE_CHARSET = /^[A-Za-z0-9_+/=-]+$/\n\n/**\n * The window an opaque secret lives in.\n *\n * Below 32 characters the false-positive rate against ordinary identifiers is\n * not worth it and the value is too short to be a modern credential anyway.\n * Above 512 it is customer CONTENT — a base64 image, a compressed canvas\n * blob, an encoded document — and blanking those is the \"gutting the export\"\n * failure this file is explicitly trying not to commit.\n */\nconst OPAQUE_MIN_LENGTH = 32\nconst OPAQUE_MAX_LENGTH = 512\n\n/** Shannon entropy per character. Prose and slugs sit well below this. */\nconst OPAQUE_MIN_ENTROPY = 3.2\n\nfunction shannonEntropyPerChar(text: string): number {\n const counts = new Map<string, number>()\n for (const character of text) {\n counts.set(character, (counts.get(character) ?? 0) + 1)\n }\n let entropy = 0\n for (const count of counts.values()) {\n const p = count / text.length\n entropy -= p * Math.log2(p)\n }\n return entropy\n}\n\n/**\n * Does this STRING look like a credential, ignoring what it is called?\n *\n * Exported because the guard spec asserts each limb independently — a shape\n * detector tested only through the redactor is a detector whose gaps are\n * invisible.\n */\nexport function looksLikeCredential(value: string): boolean {\n const text = value.trim()\n if (!text) return false\n if (PEM_PATTERN.test(text)) return true\n if (CRYPT_HASH_PATTERN.test(text)) return true\n if (SALTED_HASH_PATTERN.test(text)) return true\n if (CREDENTIAL_PREFIXES.some((prefix) => text.startsWith(prefix))) return true\n if (JWT_PATTERN.test(text)) return true\n // Anything with whitespace is prose, and `data:` is an inline asset. Both\n // are customer content, and neither is a bearer credential.\n if (/\\s/.test(text)) return false\n if (/^data:/i.test(text)) return false\n if (UUID_PATTERN.test(text)) return false\n if (text.length < OPAQUE_MIN_LENGTH || text.length > OPAQUE_MAX_LENGTH) {\n return false\n }\n if (!OPAQUE_CHARSET.test(text)) return false\n return shannonEntropyPerChar(text) >= OPAQUE_MIN_ENTROPY\n}\n\n/** What a stripped URL parameter reads as, so the removal is visible. */\nexport const REDACTED_URL_PARAM = 'REDACTED'\n\n/** Query parameters that carry a bearer credential inside a URL. */\nconst URL_CREDENTIAL_PARAMS = new Set([\n 'token',\n 'access_token',\n 'accesstoken',\n 'id_token',\n 'refresh_token',\n 'auth',\n 'authorization',\n 'apikey',\n 'api_key',\n 'key',\n 'sig',\n 'signature',\n 'secret',\n 'password',\n 'downloadtoken',\n])\n\n/**\n * Strip the credential out of a URL, keeping the URL.\n *\n * A media object's `url` is `…/o/{path}?alt=media&token={uuid}` — a live,\n * unauthenticated download credential, in a field an access request has a\n * real interest in. Blanking the whole value would tell the customer nothing\n * about where their own file is; keeping it whole hands out the credential.\n * So the parameter is replaced and the rest survives. `user:pass@` userinfo\n * goes the same way.\n *\n * Returns the input unchanged when it is not a URL — every non-URL string\n * has already been decided by the shape gate.\n */\nexport function scrubUrlCredentials(value: string): string {\n if (!/^https?:\\/\\//i.test(value)) return value\n let url: URL\n try {\n url = new URL(value)\n } catch {\n return value\n }\n let changed = false\n if (url.username || url.password) {\n url.username = ''\n url.password = ''\n changed = true\n }\n for (const name of Array.from(url.searchParams.keys())) {\n if (!URL_CREDENTIAL_PARAMS.has(name.toLowerCase())) continue\n if (!url.searchParams.get(name)) continue\n url.searchParams.set(name, REDACTED_URL_PARAM)\n changed = true\n }\n return changed ? url.toString() : value\n}\n\n/**\n * Gate (A): does the NAME say credential?\n *\n * `value` is consulted only for the metering carve-out described in the\n * header — a `tokensUsed: 14203` is a count, not a credential.\n */\nexport function secretByName(key: unknown, value?: unknown): boolean {\n const tokens = fieldNameTokens(key)\n if (!tokens.length) return false\n const hits = tokens.filter((token) => SECRET_WORDS.has(token))\n for (let index = 0; index + 1 < tokens.length; index += 1) {\n if (SECRET_WORD_PAIRS.has(`${tokens[index]} ${tokens[index + 1]}`)) {\n return true\n }\n }\n if (!hits.length) return false\n // Matched only on a metering word, over a number or a boolean: a count.\n const metering = hits.every((token) => METERING_WORDS.has(token))\n if (metering && (typeof value === 'number' || typeof value === 'boolean')) {\n return false\n }\n return true\n}\n\n/** Gate (B)'s defuser: does the NAME say this high-entropy value is an id? */\nfunction nameMarksContentIdentifier(key: unknown): boolean {\n return fieldNameTokens(key).some((token) => SAFE_VALUE_WORDS.has(token))\n}\n\n/** What a redacted value is replaced with — presence, never content. */\nexport interface RedactedValue {\n redacted: true\n /** Whether a value was actually there, so absence is distinguishable. */\n present: boolean\n reason: 'secret'\n}\n\nconst redactedMarker = (present: boolean): RedactedValue => ({\n redacted: true,\n present,\n reason: 'secret',\n})\n\n/**\n * Deep-copy a document, replacing every credential-shaped value with a\n * presence marker.\n *\n * Presence rather than deletion, because \"we hold a webhook secret for you\"\n * is itself a true and disclosable fact, and a silently-absent key would make\n * the export look like it holds less than it does — the under-disclosure this\n * feature exists to stop, reintroduced by the safety measure.\n *\n * Firestore values that are not plain JSON (Timestamp, GeoPoint, DocumentRef,\n * Buffer) are stringified rather than walked: they carry no nested keys to\n * redact, and `JSON.stringify` on a Timestamp yields `{_seconds,_nanoseconds}`,\n * which is not a date to a human reading the file.\n *\n * Three gates, any of which redacts — the name (A), the value's shape (B),\n * and a credential embedded in a URL (C). See the header above\n * {@link fieldNameTokens} for why the value half is the one that carries the\n * \"withhold unless known safe\" default.\n */\nexport function redactSecrets(value: unknown, key = ''): unknown {\n // (A) The name, tokenized — convention-independent.\n if (key && secretByName(key, value)) {\n return redactedMarker(value !== undefined && value !== null && value !== '')\n }\n if (value === null || value === undefined) return value ?? null\n if (typeof value === 'string') {\n // (C) A URL that carries its credential in a parameter keeps the URL and\n // loses the credential — see {@link scrubUrlCredentials}.\n const scrubbed = scrubUrlCredentials(value)\n if (scrubbed !== value) return scrubbed\n // (B) The shape, whatever the field is called. Withheld unless the name\n // marks it as a content identifier.\n if (looksLikeCredential(value) && !nameMarksContentIdentifier(key)) {\n return redactedMarker(true)\n }\n return value\n }\n if (Array.isArray(value)) return value.map((entry) => redactSecrets(entry, key))\n if (typeof value === 'object') {\n const source = value as Record<string, unknown>\n // Firestore Timestamp — the only non-plain value common enough to be\n // worth rendering rather than dumping.\n if (typeof (source as any).toDate === 'function') {\n try {\n return (source as any).toDate().toISOString()\n } catch {\n return String(value)\n }\n }\n if (typeof (source as any).toBase64 === 'function') return '[binary]'\n const out: Record<string, unknown> = {}\n for (const [childKey, childValue] of Object.entries(source)) {\n out[childKey] = redactSecrets(childValue, childKey)\n }\n return out\n }\n return value\n}\n\nexport interface ExportedDocument {\n /** Document id — omitted where the id is itself a secret (see `apiKeys`). */\n id?: string\n path?: string\n data: unknown\n}\n\nexport interface PersonalDataExport {\n subject: { type: ExportSubject; id: string }\n generatedAt: string\n /**\n * The disclosure decision for every source, copied into the file the\n * customer receives. An export that silently omits a collection is\n * indistinguishable from one that has nothing to report; this makes the two\n * different, and it is what a supervisory authority would ask to see.\n */\n coverage: readonly ExportSourceSpec[]\n /** Collection name → the documents disclosed for it. */\n data: Record<string, ExportedDocument[]>\n /** Collection name → row count, for sources disclosed as a count only. */\n counts: Record<string, number>\n /** Places we hold data that is not in Firestore. */\n elsewhere: readonly string[]\n}\n\n/** Data we hold outside Firestore — named, because a file that omits them reads as complete. */\nconst ELSEWHERE = [\n 'Stripe — customer object, payment methods, invoices. Request from Stripe or ask us.',\n 'Resend — transactional email delivery logs.',\n 'Google Analytics — 14-month retention, not keyed to an account.',\n 'Cloud Storage — uploaded media and avatars, under users/{uid}/, orgs/{orgId}/ and hosts/{hostId}/. Listed by path in this file; the bytes are served by their own URLs.',\n]\n\nfunction sourcesFor(subject: ExportSubject): ExportSourceSpec[] {\n return PERSONAL_DATA_SOURCES.filter((source) => source.subjects.includes(subject))\n}\n\n/** Read a top-level collection bounded by a field equal to the subject id. */\nasync function readByField(\n db: any,\n collection: string,\n field: string,\n id: string,\n options: { includeIds?: boolean } = {},\n): Promise<ExportedDocument[]> {\n const rows = await db.collection(collection).where(field, '==', id).get()\n return rows.docs.map((doc: any) => ({\n // `apiKeys` document ids are the SHA-256 of the token: an id handed back\n // is a verifier handed back, so the caller opts in rather than out.\n ...(options.includeIds === false ? {} : { id: doc.id }),\n data: redactSecrets(doc.data()),\n }))\n}\n\n/** Read one document by id, or nothing when it does not exist. */\nasync function readDoc(\n db: any,\n collection: string,\n id: string,\n): Promise<ExportedDocument[]> {\n const snapshot = await db.collection(collection).doc(id).get()\n if (!snapshot.exists) return []\n return [{ id, data: redactSecrets(snapshot.data()) }]\n}\n\n/**\n * Walk a document and everything beneath it.\n *\n * `listCollections()` rather than a hard-coded subcollection list, for the\n * same reason the erasure uses `recursiveDelete`: a subcollection added next\n * quarter is reached without anybody remembering this file exists. Bounded by\n * `maxDepth` because the tree is customer-shaped and an unbounded recursion\n * inside a request is how AGL-1455 lost erasures to a 60-second timeout.\n */\nasync function readSubtree(\n db: any,\n ref: any,\n depth = 0,\n maxDepth = 4,\n): Promise<ExportedDocument[]> {\n const snapshot = await ref.get()\n const found: ExportedDocument[] = []\n if (snapshot.exists) {\n found.push({ path: ref.path, data: redactSecrets(snapshot.data()) })\n }\n if (depth >= maxDepth) return found\n const children = (await ref.listCollections?.()) ?? []\n for (const child of children) {\n const rows = await child.get()\n for (const doc of rows.docs) {\n found.push(...(await readSubtree(db, doc.ref, depth + 1, maxDepth)))\n }\n }\n return found\n}\n\nexport interface ExportOptions {\n /** Injectable for tests; defaults to the admin app's Firestore. */\n firestore?: any\n /** Stops a pathological workspace from building an unbounded file. */\n maxDepth?: number\n}\n\n/**\n * Everything we hold about a PERSON, as the person themselves may ask for it\n * (GDPR Art. 15 and Art. 20).\n *\n * Scoped to them, never to the workspaces they belong to. Their membership row\n * in each org is theirs; the roster around it is their colleagues'. An org\n * export is a separate call with a separate authorization, which is the whole\n * reason the two are different functions rather than one with a flag.\n */\nexport async function exportUserData(\n uid: string,\n options: ExportOptions = {},\n): Promise<PersonalDataExport> {\n const db = options.firestore ?? firestoreApp()\n const data: Record<string, ExportedDocument[]> = {}\n const counts: Record<string, number> = {}\n\n // The profile tree — `users/{uid}` and every subcollection under it.\n data['users'] = await readSubtree(\n db,\n db.collection('users').doc(uid),\n 0,\n options.maxDepth ?? 4,\n )\n\n // The public identity, which the user tree does not reach (AGL-1970).\n data['profiles'] = await readDoc(db, 'profiles', uid)\n\n // ONLY the messages this person wrote. The threads belong to a workspace\n // and carry other people's words; handing a member the whole conversation\n // to answer their own access request is a disclosure nobody consented to.\n const messages = await db\n .collectionGroup('messages')\n .where('authorId', '==', uid)\n .get()\n .catch(() => ({ docs: [] as any[] }))\n data['supportTickets'] = messages.docs.map((doc: any) => ({\n path: doc.ref.path,\n data: redactSecrets(doc.data()),\n }))\n\n return {\n subject: { type: 'user', id: uid },\n generatedAt: new Date().toISOString(),\n coverage: sourcesFor('user'),\n data,\n counts,\n elsewhere: ELSEWHERE,\n }\n}\n\n/**\n * Everything we hold about a WORKSPACE, for its owner (Art. 15 for the\n * personal data in it, and the \"export before you delete\" the Delete tab,\n * DPA §11 and Terms §13.3 have been promising).\n *\n * Every top-level collection the erasure sweeps by `orgId` is read here by the\n * same bound, so the two see the same rows. Where a source is withheld, its\n * COUNT is still reported: a collection that appears in neither the data nor\n * the counts is indistinguishable from one that does not exist, and that\n * ambiguity is what makes an incomplete export undetectable.\n */\nexport async function exportOrgData(\n orgId: string,\n options: ExportOptions = {},\n): Promise<PersonalDataExport> {\n const db = options.firestore ?? firestoreApp()\n const data: Record<string, ExportedDocument[]> = {}\n const counts: Record<string, number> = {}\n\n data['orgs'] = await readSubtree(\n db,\n db.collection('orgs').doc(orgId),\n 0,\n options.maxDepth ?? 4,\n )\n\n // Sites: the routing entry and the whole tree for each.\n const hosts = await db.collection('hosts').where('orgId', '==', orgId).get()\n const hostDocs: ExportedDocument[] = []\n const routing: ExportedDocument[] = []\n const outbox: ExportedDocument[] = []\n for (const host of hosts.docs) {\n hostDocs.push(...(await readSubtree(db, host.ref, 0, options.maxDepth ?? 4)))\n routing.push(...(await readDoc(db, 'hostIndex', host.id)))\n // Keyed by `hostId`, not `orgId`, so it is read here rather than in the\n // field-keyed loop below — the same bound `eraseHostSupplierDeliveries`\n // sweeps by, so the export and the erasure see the same rows.\n outbox.push(...(await readByField(db, 'supplierDeliveries', 'hostId', host.id)))\n }\n data['hosts'] = hostDocs\n data['hostIndex'] = routing\n data['supplierDeliveries'] = outbox\n\n // The field-keyed collections a path cascade is blind to — the ones the\n // erasure had to grow a sweep for, one issue at a time.\n for (const collection of [\n 'ssoDomains',\n CONSOLE_DOMAINS_COLLECTION,\n 'stripeCustomers',\n 'orgSlugs',\n 'publisherHandles',\n 'platformRevenue',\n 'storefrontTaxCollected',\n ]) {\n data[collection] = await readByField(db, collection, 'orgId', orgId)\n }\n\n // Existence, never the credential — and not the id either, which is the\n // token's SHA-256.\n data['apiKeys'] = await readByField(db, 'apiKeys', 'orgId', orgId, {\n includeIds: false,\n })\n\n // Which mailboxes the reps connected, never the grant behind one: the\n // tokens are withheld by `redactSecrets` like every other read here. The id\n // is the mailbox's own, so unlike `apiKeys` it stays.\n data['outreachMailboxCredentials'] = await readByField(\n db,\n 'outreachMailboxCredentials',\n 'orgId',\n orgId,\n )\n\n // Where each short tracking link the org's sequences sent points (AGL-3297).\n data['outreachLinks'] = await readByField(db, 'outreachLinks', 'orgId', orgId)\n\n // Shipping (AGL-3612), tax services (AGL-3631), marketing platforms\n // (AGL-3639) and help desks (AGL-3824): top-level and keyed by `orgId`, the bound their\n // `orgKeyedCollections` erasure sweeps by. The sealed credentials are\n // withheld by `redactSecrets` on their names.\n for (const collection of [\n 'shippingTrackers',\n 'shippingQuoteCache',\n 'taxEngineConnections',\n 'taxEngineProductCodes',\n 'taxEngineExemptions',\n 'taxEngineTransactions',\n 'marketingPlatformConnections',\n 'marketingPlatformEvents',\n 'helpDeskConnections',\n 'helpDeskOutbox',\n 'helpDeskCustomers',\n 'adConversionConnections',\n 'adConversionConsents',\n 'adConversionEvents',\n 'fulfillmentNetworkConnections',\n 'fulfillmentNetworkOrders',\n 'courierConnections',\n 'courierDeliveries',\n 'marketplaceConnections',\n 'marketplaceOrders',\n 'inventorySyncConnections',\n 'inventorySyncOrders',\n 'inventorySyncProducts',\n 'podConnections',\n 'podProductLinks',\n 'podOrders',\n 'deliveryAppStores',\n 'deliveryAppOrders',\n ]) {\n data[collection] = await readByField(db, collection, 'orgId', orgId)\n }\n\n data['publisherProfiles'] = await readDoc(db, 'publisherProfiles', orgId)\n data['marketplaceListings'] = await readByField(\n db,\n 'marketplaceListings',\n 'profileId',\n orgId,\n )\n\n // Support threads WITH their messages — for an org the whole thread is the\n // org's own record, unlike the person-scoped export above.\n const tickets = await db\n .collection('supportTickets')\n .where('orgId', '==', orgId)\n .get()\n const ticketDocs: ExportedDocument[] = []\n for (const ticket of tickets.docs) {\n ticketDocs.push(...(await readSubtree(db, ticket.ref, 0, 2)))\n }\n data['supportTickets'] = ticketDocs\n\n // Withheld, but counted — see the note on this function.\n const replayKeys = await db\n .collection('apiIdempotency')\n .where('orgId', '==', orgId)\n .get()\n counts['apiIdempotency'] = replayKeys.docs.length\n\n return {\n subject: { type: 'org', id: orgId },\n generatedAt: new Date().toISOString(),\n coverage: sourcesFor('org'),\n data,\n counts,\n elsewhere: ELSEWHERE,\n }\n}\n\n/** A filename a human can file: subject, id and the date it was produced. */\nexport function exportFilename(exported: PersonalDataExport): string {\n const day = exported.generatedAt.slice(0, 10)\n return `aglyn-${exported.subject.type}-data-${exported.subject.id}-${day}.json`\n}\n\nexport default exportUserData\n"],"names":["firebaseAdmin","CONSOLE_DOMAINS_COLLECTION","firestoreApp","app","firestore","PERSONAL_DATA_SOURCES","collection","keyedBy","subjects","exported","note","EXPORT_WITHHELD","filter","s","EXPORT_COVERED_COLLECTIONS","map","source","fieldNameTokens","key","String","replace","toLowerCase","split","Boolean","SECRET_WORDS","Set","SECRET_WORD_PAIRS","METERING_WORDS","SAFE_VALUE_WORDS","CREDENTIAL_PREFIXES","PEM_PATTERN","CRYPT_HASH_PATTERN","SALTED_HASH_PATTERN","JWT_PATTERN","UUID_PATTERN","OPAQUE_CHARSET","OPAQUE_MIN_LENGTH","OPAQUE_MAX_LENGTH","OPAQUE_MIN_ENTROPY","shannonEntropyPerChar","text","counts","Map","character","set","get","entropy","count","values","p","length","Math","log2","looksLikeCredential","value","trim","test","some","prefix","startsWith","REDACTED_URL_PARAM","URL_CREDENTIAL_PARAMS","scrubUrlCredentials","url","URL","changed","username","password","name","Array","from","searchParams","keys","has","toString","secretByName","tokens","hits","token","index","metering","every","nameMarksContentIdentifier","redactedMarker","present","redacted","reason","redactSecrets","undefined","scrubbed","isArray","entry","toDate","toISOString","toBase64","out","childKey","childValue","Object","entries","ELSEWHERE","sourcesFor","subject","includes","readByField","db","field","id","options","rows","where","docs","doc","includeIds","data","readDoc","snapshot","exists","readSubtree","ref","depth","maxDepth","found","push","path","children","listCollections","child","exportUserData","uid","messages","collectionGroup","catch","type","generatedAt","Date","coverage","elsewhere","exportOrgData","orgId","hosts","hostDocs","routing","outbox","host","tickets","ticketDocs","ticket","replayKeys","exportFilename","day","slice"],"mappings":";AAAA;;;;;;;;;;;;;;;CAeC,GAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA0DC,GAED,SAASA,aAAa,QAAQ,sBAAkB;AAChD,SAASC,0BAA0B,QAAQ,uBAAmB;AAE9D,MAAMC,eAAe,IAAMF,cAAcG,GAAG,GAAGC,SAAS;AAiCxD;;;;;;CAMC,GACD,OAAO,MAAMC,wBAAqD;IAChE,yEAAyE;IACzE;QACEC,YAAY;QACZC,SAAS;QACTC,UAAU;YAAC;SAAO;QAClBC,UAAU;QACVC,MAAM;IACR;IACA;QACEJ,YAAY;QACZC,SAAS;QACTC,UAAU;YAAC;SAAO;QAClBC,UAAU;QACVC,MAAM;IACR;IACA,yEAAyE;IACzE;QACEJ,YAAY;QACZC,SAAS;QACTC,UAAU;YAAC;SAAM;QACjBC,UAAU;QACVC,MAAM;IACR;IACA;QACEJ,YAAY;QACZC,SAAS;QACTC,UAAU;YAAC;SAAM;QACjBC,UAAU;QACVC,MAAM;IACR;IACA;QACEJ,YAAY;QACZC,SAAS;QACTC,UAAU;YAAC;SAAM;QACjBC,UAAU;QACVC,MAAM;IACR;IACA;QACEJ,YAAY;QACZC,SAAS;QACTC,UAAU;YAAC;SAAM;QACjBC,UAAU;QACVC,MAAM;IACR;IACA,yEAAyE;IACzE;QACEJ,YAAY;QACZC,SAAS;QACTC,UAAU;YAAC;SAAM;QACjBC,UAAU;QACVC,MAAM;IACR;IACA;QACEJ,YAAY;QACZC,SAAS;QACTC,UAAU;YAAC;SAAM;QACjBC,UAAU;QACVC,MAAM;IACR;IACA;QACEJ,YAAY;QACZC,SAAS;QACTC,UAAU;YAAC;SAAM;QACjBC,UAAU;QACVC,MAAM;IACR;IACA;QACEJ,YAAY;QACZC,SAAS;QACTC,UAAU;YAAC;SAAM;QACjBC,UAAU;QACVC,MAAM;IACR;IACA;QACEJ,YAAY;QACZC,SAAS;QACTC,UAAU;YAAC;SAAM;QACjBC,UAAU;QACVC,MAAM;IACR;IACA;QACEJ,YAAY;QACZC,SAAS;QACTC,UAAU;YAAC;SAAM;QACjBC,UAAU;QACVC,MAAM;IACR;IACA;QACEJ,YAAY;QACZC,SAAS;QACTC,UAAU;YAAC;SAAM;QACjBC,UAAU;QACVC,MAAM;IACR;IACA;QACEJ,YAAY;QACZC,SAAS;QACTC,UAAU;YAAC;SAAM;QACjBC,UAAU;QACVC,MAAM;IACR;IACA;QACEJ,YAAY;QACZC,SAAS;QACTC,UAAU;YAAC;SAAM;QACjBC,UAAU;QACVC,MAAM;IACR;IACA;QACEJ,YAAY;QACZC,SAAS;QACTC,UAAU;YAAC;SAAM;QACjBC,UAAU;QACVC,MAAM;IACR;IACA;QACEJ,YAAY;QACZC,SAAS;QACTC,UAAU;YAAC;SAAM;QACjBC,UAAU;QACVC,MAAM;IACR;IACA;QACEJ,YAAY;QACZC,SAAS;QACTC,UAAU;YAAC;SAAM;QACjBC,UAAU;QACVC,MAAM;IACR;IACA;QACEJ,YAAY;QACZC,SAAS;QACTC,UAAU;YAAC;SAAM;QACjBC,UAAU;QACVC,MAAM;IACR;IACA;QACEJ,YAAY;QACZC,SAAS;QACTC,UAAU;YAAC;SAAM;QACjBC,UAAU;QACVC,MAAM;IACR;IACA;QACEJ,YAAY;QACZC,SAAS;QACTC,UAAU;YAAC;SAAM;QACjBC,UAAU;QACVC,MAAM;IACR;IACA;QACEJ,YAAY;QACZC,SAAS;QACTC,UAAU;YAAC;SAAM;QACjBC,UAAU;QACVC,MAAM;IACR;IACA;QACEJ,YAAY;QACZC,SAAS;QACTC,UAAU;YAAC;SAAM;QACjBC,UAAU;QACVC,MAAM;IACR;IACA;QACEJ,YAAY;QACZC,SAAS;QACTC,UAAU;YAAC;SAAM;QACjBC,UAAU;QACVC,MAAM;IACR;IACA;QACEJ,YAAY;QACZC,SAAS;QACTC,UAAU;YAAC;SAAM;QACjBC,UAAU;QACVC,MAAM;IACR;IACA;QACEJ,YAAY;QACZC,SAAS;QACTC,UAAU;YAAC;SAAM;QACjBC,UAAU;QACVC,MAAM;IACR;IACA;QACEJ,YAAY;QACZC,SAAS;QACTC,UAAU;YAAC;SAAM;QACjBC,UAAU;QACVC,MAAM;IACR;IACA;QACEJ,YAAY;QACZC,SAAS;QACTC,UAAU;YAAC;SAAM;QACjBC,UAAU;QACVC,MAAM;IACR;IACA;QACEJ,YAAY;QACZC,SAAS;QACTC,UAAU;YAAC;SAAM;QACjBC,UAAU;QACVC,MAAM;IACR;IACA;QACEJ,YAAY;QACZC,SAAS;QACTC,UAAU;YAAC;SAAM;QACjBC,UAAU;QACVC,MAAM;IACR;IACA;QACEJ,YAAY;QACZC,SAAS;QACTC,UAAU;YAAC;SAAM;QACjBC,UAAU;QACVC,MAAM;IACR;IACA;QACEJ,YAAY;QACZC,SAAS;QACTC,UAAU;YAAC;SAAM;QACjBC,UAAU;QACVC,MAAM;IACR;IACA;QACEJ,YAAY;QACZC,SAAS;QACTC,UAAU;YAAC;SAAM;QACjBC,UAAU;QACVC,MAAM;IACR;IACA;QACEJ,YAAY;QACZC,SAAS;QACTC,UAAU;YAAC;SAAM;QACjBC,UAAU;QACVC,MAAM;IACR;IACA;QACEJ,YAAY;QACZC,SAAS;QACTC,UAAU;YAAC;SAAM;QACjBC,UAAU;QACVC,MAAM;IACR;IACA;QACEJ,YAAY;QACZC,SAAS;QACTC,UAAU;YAAC;SAAM;QACjBC,UAAU;QACVC,MAAM;IACR;IACA;QACEJ,YAAY;QACZC,SAAS;QACTC,UAAU;YAAC;SAAM;QACjBC,UAAU;QACVC,MAAM;IACR;IACA;QACEJ,YAAY;QACZC,SAAS;QACTC,UAAU;YAAC;SAAM;QACjBC,UAAU;QACVC,MAAM;IACR;IACA;QACEJ,YAAY;QACZC,SAAS;QACTC,UAAU;YAAC;SAAM;QACjBC,UAAU;QACVC,MAAM;IACR;IACA;QACEJ,YAAY;QACZC,SAAS;QACTC,UAAU;YAAC;SAAM;QACjBC,UAAU;QACVC,MAAM;IACR;IACA;QACEJ,YAAYL;QACZM,SAAS;QACTC,UAAU;YAAC;SAAM;QACjBC,UAAU;QACVC,MAAM;IACR;IACA;QACEJ,YAAY;QACZC,SAAS;QACTC,UAAU;YAAC;SAAM;QACjBC,UAAU;QACVC,MAAM;IACR;IACA;QACEJ,YAAY;QACZC,SAAS;QACTC,UAAU;YAAC;SAAM;QACjBC,UAAU;QACVC,MAAM;IACR;IACA;QACEJ,YAAY;QACZC,SAAS;QACTC,UAAU;YAAC;SAAM;QACjBC,UAAU;QACVC,MAAM;IACR;IACA;QACEJ,YAAY;QACZC,SAAS;QACTC,UAAU;YAAC;SAAM;QACjBC,UAAU;QACVC,MAAM;IACR;IACA;QACEJ,YAAY;QACZC,SAAS;QACTC,UAAU;YAAC;SAAM;QACjBC,UAAU;QACVC,MAAM;IACR;IACA;QACEJ,YAAY;QACZC,SAAS;QACTC,UAAU;YAAC;SAAM;QACjBC,UAAU;QACVC,MAAM;IACR;IACA;QACEJ,YAAY;QACZC,SAAS;QACTC,UAAU;YAAC;YAAO;SAAO;QACzBC,UAAU;QACVC,MAAM;IACR;IACA,yEAAyE;IACzE;QACEJ,YAAY;QACZC,SAAS;QACTC,UAAU;YAAC;SAAM;QACjBC,UAAU;QACVC,MAAM;IACR;IACA;QACEJ,YAAY;QACZC,SAAS;QACTC,UAAU;YAAC;SAAM;QACjBC,UAAU;QACVC,MAAM;IACR;IACA,yEAAyE;IACzE;QACEJ,YAAY;QACZC,SAAS;QACTC,UAAU;YAAC;YAAO;SAAO;QACzBC,UAAU;QACVC,MAAM;IACR;CACD,CAAA;AAED,iEAAiE,GACjE,OAAO,MAAMC,kBAAkBN,sBAAsBO,MAAM,CAAC,CAACC,IAAM,CAACA,EAAEJ,QAAQ,EAAC;AAE/E,yEAAyE,GACzE,OAAO,MAAMK,6BACXT,sBAAsBU,GAAG,CAAC,CAACC,SAAWA,OAAOV,UAAU,EAAC;AAE1D;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA4DC,GAED;;;;;;;CAOC,GACD,OAAO,SAASW,gBAAgBC,GAAY;IAC1C,OAAOC,OAAOD,cAAAA,MAAO,IAClBE,OAAO,CAAC,kBAAkB,KAC1BA,OAAO,CAAC,sBAAsB,SAC9BA,OAAO,CAAC,yBAAyB,SACjCA,OAAO,CAAC,sBAAsB,SAC9BC,WAAW,GACXC,KAAK,CAAC,KACNV,MAAM,CAACW;AACZ;AAEA;;;;;;;;;CASC,GACD,MAAMC,eAAe,IAAIC,IAAI;IAC3B;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;CACD;AAED;;;;;;;CAOC,GACD,MAAMC,oBAAoB,IAAID,IAAI;IAChC;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA,wEAAwE;IACxE,yEAAyE;IACzE;IACA,yEAAyE;IACzE,wEAAwE;IACxE,2CAA2C;IAC3C;IACA;IACA;IACA;CACD;AAED;;;CAGC,GACD,MAAME,iBAAiB,IAAIF,IAAI;IAAC;IAAS;CAAS;AAElD;;;;;;;;CAQC,GACD,MAAMG,mBAAmB,IAAIH,IAAI;IAC/B,4EAA4E;IAC5E,uEAAuE;IACvE,+DAA+D;IAC/D;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;CACD;AAED,2EAA2E,GAC3E,MAAMI,sBAAsB;IAC1B;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;CACD;AAED,qEAAqE,GACrE,MAAMC,cAAc;AAEpB,wEAAwE,GACxE,MAAMC,qBAAqB;AAE3B,0EAA0E,GAC1E,MAAMC,sBAAsB;AAE5B,iDAAiD,GACjD,MAAMC,cAAc;AAEpB,2EAA2E,GAC3E,MAAMC,eACJ;AAEF,6EAA6E,GAC7E,MAAMC,iBAAiB;AAEvB;;;;;;;;CAQC,GACD,MAAMC,oBAAoB;AAC1B,MAAMC,oBAAoB;AAE1B,wEAAwE,GACxE,MAAMC,qBAAqB;AAE3B,SAASC,sBAAsBC,IAAY;IACzC,MAAMC,SAAS,IAAIC;IACnB,KAAK,MAAMC,aAAaH,KAAM;YACLC;QAAvBA,OAAOG,GAAG,CAACD,WAAW,EAACF,cAAAA,OAAOI,GAAG,CAACF,sBAAXF,cAAyB,KAAK;IACvD;IACA,IAAIK,UAAU;IACd,KAAK,MAAMC,SAASN,OAAOO,MAAM,GAAI;QACnC,MAAMC,IAAIF,QAAQP,KAAKU,MAAM;QAC7BJ,WAAWG,IAAIE,KAAKC,IAAI,CAACH;IAC3B;IACA,OAAOH;AACT;AAEA;;;;;;CAMC,GACD,OAAO,SAASO,oBAAoBC,KAAa;IAC/C,MAAMd,OAAOc,MAAMC,IAAI;IACvB,IAAI,CAACf,MAAM,OAAO;IAClB,IAAIV,YAAY0B,IAAI,CAAChB,OAAO,OAAO;IACnC,IAAIT,mBAAmByB,IAAI,CAAChB,OAAO,OAAO;IAC1C,IAAIR,oBAAoBwB,IAAI,CAAChB,OAAO,OAAO;IAC3C,IAAIX,oBAAoB4B,IAAI,CAAC,CAACC,SAAWlB,KAAKmB,UAAU,CAACD,UAAU,OAAO;IAC1E,IAAIzB,YAAYuB,IAAI,CAAChB,OAAO,OAAO;IACnC,0EAA0E;IAC1E,4DAA4D;IAC5D,IAAI,KAAKgB,IAAI,CAAChB,OAAO,OAAO;IAC5B,IAAI,UAAUgB,IAAI,CAAChB,OAAO,OAAO;IACjC,IAAIN,aAAasB,IAAI,CAAChB,OAAO,OAAO;IACpC,IAAIA,KAAKU,MAAM,GAAGd,qBAAqBI,KAAKU,MAAM,GAAGb,mBAAmB;QACtE,OAAO;IACT;IACA,IAAI,CAACF,eAAeqB,IAAI,CAAChB,OAAO,OAAO;IACvC,OAAOD,sBAAsBC,SAASF;AACxC;AAEA,uEAAuE,GACvE,OAAO,MAAMsB,qBAAqB,WAAU;AAE5C,kEAAkE,GAClE,MAAMC,wBAAwB,IAAIpC,IAAI;IACpC;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;CACD;AAED;;;;;;;;;;;;CAYC,GACD,OAAO,SAASqC,oBAAoBR,KAAa;IAC/C,IAAI,CAAC,gBAAgBE,IAAI,CAACF,QAAQ,OAAOA;IACzC,IAAIS;IACJ,IAAI;QACFA,MAAM,IAAIC,IAAIV;IAChB,EAAE,eAAM;QACN,OAAOA;IACT;IACA,IAAIW,UAAU;IACd,IAAIF,IAAIG,QAAQ,IAAIH,IAAII,QAAQ,EAAE;QAChCJ,IAAIG,QAAQ,GAAG;QACfH,IAAII,QAAQ,GAAG;QACfF,UAAU;IACZ;IACA,KAAK,MAAMG,QAAQC,MAAMC,IAAI,CAACP,IAAIQ,YAAY,CAACC,IAAI,IAAK;QACtD,IAAI,CAACX,sBAAsBY,GAAG,CAACL,KAAK/C,WAAW,KAAK;QACpD,IAAI,CAAC0C,IAAIQ,YAAY,CAAC1B,GAAG,CAACuB,OAAO;QACjCL,IAAIQ,YAAY,CAAC3B,GAAG,CAACwB,MAAMR;QAC3BK,UAAU;IACZ;IACA,OAAOA,UAAUF,IAAIW,QAAQ,KAAKpB;AACpC;AAEA;;;;;CAKC,GACD,OAAO,SAASqB,aAAazD,GAAY,EAAEoC,KAAe;IACxD,MAAMsB,SAAS3D,gBAAgBC;IAC/B,IAAI,CAAC0D,OAAO1B,MAAM,EAAE,OAAO;IAC3B,MAAM2B,OAAOD,OAAOhE,MAAM,CAAC,CAACkE,QAAUtD,aAAaiD,GAAG,CAACK;IACvD,IAAK,IAAIC,QAAQ,GAAGA,QAAQ,IAAIH,OAAO1B,MAAM,EAAE6B,SAAS,EAAG;QACzD,IAAIrD,kBAAkB+C,GAAG,CAAC,GAAGG,MAAM,CAACG,MAAM,CAAC,CAAC,EAAEH,MAAM,CAACG,QAAQ,EAAE,EAAE,GAAG;YAClE,OAAO;QACT;IACF;IACA,IAAI,CAACF,KAAK3B,MAAM,EAAE,OAAO;IACzB,wEAAwE;IACxE,MAAM8B,WAAWH,KAAKI,KAAK,CAAC,CAACH,QAAUnD,eAAe8C,GAAG,CAACK;IAC1D,IAAIE,YAAa,CAAA,OAAO1B,UAAU,YAAY,OAAOA,UAAU,SAAQ,GAAI;QACzE,OAAO;IACT;IACA,OAAO;AACT;AAEA,4EAA4E,GAC5E,SAAS4B,2BAA2BhE,GAAY;IAC9C,OAAOD,gBAAgBC,KAAKuC,IAAI,CAAC,CAACqB,QAAUlD,iBAAiB6C,GAAG,CAACK;AACnE;AAUA,MAAMK,iBAAiB,CAACC,UAAqC,CAAA;QAC3DC,UAAU;QACVD;QACAE,QAAQ;IACV,CAAA;AAEA;;;;;;;;;;;;;;;;;;CAkBC,GACD,OAAO,SAASC,cAAcjC,KAAc,EAAEpC,MAAM,EAAE;IACpD,oDAAoD;IACpD,IAAIA,OAAOyD,aAAazD,KAAKoC,QAAQ;QACnC,OAAO6B,eAAe7B,UAAUkC,aAAalC,UAAU,QAAQA,UAAU;IAC3E;IACA,IAAIA,UAAU,QAAQA,UAAUkC,WAAW,OAAOlC,gBAAAA,QAAS;IAC3D,IAAI,OAAOA,UAAU,UAAU;QAC7B,yEAAyE;QACzE,0DAA0D;QAC1D,MAAMmC,WAAW3B,oBAAoBR;QACrC,IAAImC,aAAanC,OAAO,OAAOmC;QAC/B,wEAAwE;QACxE,oCAAoC;QACpC,IAAIpC,oBAAoBC,UAAU,CAAC4B,2BAA2BhE,MAAM;YAClE,OAAOiE,eAAe;QACxB;QACA,OAAO7B;IACT;IACA,IAAIe,MAAMqB,OAAO,CAACpC,QAAQ,OAAOA,MAAMvC,GAAG,CAAC,CAAC4E,QAAUJ,cAAcI,OAAOzE;IAC3E,IAAI,OAAOoC,UAAU,UAAU;QAC7B,MAAMtC,SAASsC;QACf,qEAAqE;QACrE,uCAAuC;QACvC,IAAI,OAAO,AAACtC,OAAe4E,MAAM,KAAK,YAAY;YAChD,IAAI;gBACF,OAAO,AAAC5E,OAAe4E,MAAM,GAAGC,WAAW;YAC7C,EAAE,eAAM;gBACN,OAAO1E,OAAOmC;YAChB;QACF;QACA,IAAI,OAAO,AAACtC,OAAe8E,QAAQ,KAAK,YAAY,OAAO;QAC3D,MAAMC,MAA+B,CAAC;QACtC,KAAK,MAAM,CAACC,UAAUC,WAAW,IAAIC,OAAOC,OAAO,CAACnF,QAAS;YAC3D+E,GAAG,CAACC,SAAS,GAAGT,cAAcU,YAAYD;QAC5C;QACA,OAAOD;IACT;IACA,OAAOzC;AACT;AA2BA,8FAA8F,GAC9F,MAAM8C,YAAY;IAChB;IACA;IACA;IACA;CACD;AAED,SAASC,WAAWC,OAAsB;IACxC,OAAOjG,sBAAsBO,MAAM,CAAC,CAACI,SAAWA,OAAOR,QAAQ,CAAC+F,QAAQ,CAACD;AAC3E;AAEA,4EAA4E,GAC5E,eAAeE,YACbC,EAAO,EACPnG,UAAkB,EAClBoG,KAAa,EACbC,EAAU,EACVC,UAAoC,CAAC,CAAC;IAEtC,MAAMC,OAAO,MAAMJ,GAAGnG,UAAU,CAACA,YAAYwG,KAAK,CAACJ,OAAO,MAAMC,IAAI9D,GAAG;IACvE,OAAOgE,KAAKE,IAAI,CAAChG,GAAG,CAAC,CAACiG,MAAc,aAG9BJ,QAAQK,UAAU,KAAK,QAAQ,CAAC,IAAI;YAAEN,IAAIK,IAAIL,EAAE;QAAC;YACrDO,MAAM3B,cAAcyB,IAAIE,IAAI;;AAEhC;AAEA,gEAAgE,GAChE,eAAeC,QACbV,EAAO,EACPnG,UAAkB,EAClBqG,EAAU;IAEV,MAAMS,WAAW,MAAMX,GAAGnG,UAAU,CAACA,YAAY0G,GAAG,CAACL,IAAI9D,GAAG;IAC5D,IAAI,CAACuE,SAASC,MAAM,EAAE,OAAO,EAAE;IAC/B,OAAO;QAAC;YAAEV;YAAIO,MAAM3B,cAAc6B,SAASF,IAAI;QAAI;KAAE;AACvD;AAEA;;;;;;;;CAQC,GACD,eAAeI,YACbb,EAAO,EACPc,GAAQ,EACRC,QAAQ,CAAC,EACTC,WAAW,CAAC;QAQM;IANlB,MAAML,WAAW,MAAMG,IAAI1E,GAAG;IAC9B,MAAM6E,QAA4B,EAAE;IACpC,IAAIN,SAASC,MAAM,EAAE;QACnBK,MAAMC,IAAI,CAAC;YAAEC,MAAML,IAAIK,IAAI;YAAEV,MAAM3B,cAAc6B,SAASF,IAAI;QAAI;IACpE;IACA,IAAIM,SAASC,UAAU,OAAOC;IAC9B,MAAMG,YAAY,OAAA,OAAMN,IAAIO,eAAe,oBAAnBP,IAAIO,eAAe,MAAnBP,iBAAN,OAAkC,EAAE;IACtD,KAAK,MAAMQ,SAASF,SAAU;QAC5B,MAAMhB,OAAO,MAAMkB,MAAMlF,GAAG;QAC5B,KAAK,MAAMmE,OAAOH,KAAKE,IAAI,CAAE;YAC3BW,MAAMC,IAAI,IAAK,MAAML,YAAYb,IAAIO,IAAIO,GAAG,EAAEC,QAAQ,GAAGC;QAC3D;IACF;IACA,OAAOC;AACT;AASA;;;;;;;;CAQC,GACD,OAAO,eAAeM,eACpBC,GAAW,EACXrB,UAAyB,CAAC,CAAC;QAEhBA,oBASTA;IATF,MAAMH,MAAKG,qBAAAA,QAAQxG,SAAS,YAAjBwG,qBAAqB1G;IAChC,MAAMgH,OAA2C,CAAC;IAClD,MAAMzE,SAAiC,CAAC;IAExC,qEAAqE;IACrEyE,IAAI,CAAC,QAAQ,GAAG,MAAMI,YACpBb,IACAA,GAAGnG,UAAU,CAAC,SAAS0G,GAAG,CAACiB,MAC3B,IACArB,oBAAAA,QAAQa,QAAQ,YAAhBb,oBAAoB;IAGtB,sEAAsE;IACtEM,IAAI,CAAC,WAAW,GAAG,MAAMC,QAAQV,IAAI,YAAYwB;IAEjD,yEAAyE;IACzE,0EAA0E;IAC1E,0EAA0E;IAC1E,MAAMC,WAAW,MAAMzB,GACpB0B,eAAe,CAAC,YAChBrB,KAAK,CAAC,YAAY,MAAMmB,KACxBpF,GAAG,GACHuF,KAAK,CAAC,IAAO,CAAA;YAAErB,MAAM,EAAE;QAAU,CAAA;IACpCG,IAAI,CAAC,iBAAiB,GAAGgB,SAASnB,IAAI,CAAChG,GAAG,CAAC,CAACiG,MAAc,CAAA;YACxDY,MAAMZ,IAAIO,GAAG,CAACK,IAAI;YAClBV,MAAM3B,cAAcyB,IAAIE,IAAI;QAC9B,CAAA;IAEA,OAAO;QACLZ,SAAS;YAAE+B,MAAM;YAAQ1B,IAAIsB;QAAI;QACjCK,aAAa,IAAIC,OAAO1C,WAAW;QACnC2C,UAAUnC,WAAW;QACrBa;QACAzE;QACAgG,WAAWrC;IACb;AACF;AAEA;;;;;;;;;;CAUC,GACD,OAAO,eAAesC,cACpBC,KAAa,EACb/B,UAAyB,CAAC,CAAC;QAEhBA,oBAQTA;IARF,MAAMH,MAAKG,qBAAAA,QAAQxG,SAAS,YAAjBwG,qBAAqB1G;IAChC,MAAMgH,OAA2C,CAAC;IAClD,MAAMzE,SAAiC,CAAC;IAExCyE,IAAI,CAAC,OAAO,GAAG,MAAMI,YACnBb,IACAA,GAAGnG,UAAU,CAAC,QAAQ0G,GAAG,CAAC2B,QAC1B,IACA/B,oBAAAA,QAAQa,QAAQ,YAAhBb,oBAAoB;IAGtB,wDAAwD;IACxD,MAAMgC,QAAQ,MAAMnC,GAAGnG,UAAU,CAAC,SAASwG,KAAK,CAAC,SAAS,MAAM6B,OAAO9F,GAAG;IAC1E,MAAMgG,WAA+B,EAAE;IACvC,MAAMC,UAA8B,EAAE;IACtC,MAAMC,SAA6B,EAAE;IACrC,KAAK,MAAMC,QAAQJ,MAAM7B,IAAI,CAAE;YACwBH;QAArDiC,SAASlB,IAAI,IAAK,MAAML,YAAYb,IAAIuC,KAAKzB,GAAG,EAAE,IAAGX,qBAAAA,QAAQa,QAAQ,YAAhBb,qBAAoB;QACzEkC,QAAQnB,IAAI,IAAK,MAAMR,QAAQV,IAAI,aAAauC,KAAKrC,EAAE;QACvD,wEAAwE;QACxE,wEAAwE;QACxE,8DAA8D;QAC9DoC,OAAOpB,IAAI,IAAK,MAAMnB,YAAYC,IAAI,sBAAsB,UAAUuC,KAAKrC,EAAE;IAC/E;IACAO,IAAI,CAAC,QAAQ,GAAG2B;IAChB3B,IAAI,CAAC,YAAY,GAAG4B;IACpB5B,IAAI,CAAC,qBAAqB,GAAG6B;IAE7B,wEAAwE;IACxE,wDAAwD;IACxD,KAAK,MAAMzI,cAAc;QACvB;QACAL;QACA;QACA;QACA;QACA;QACA;KACD,CAAE;QACDiH,IAAI,CAAC5G,WAAW,GAAG,MAAMkG,YAAYC,IAAInG,YAAY,SAASqI;IAChE;IAEA,wEAAwE;IACxE,mBAAmB;IACnBzB,IAAI,CAAC,UAAU,GAAG,MAAMV,YAAYC,IAAI,WAAW,SAASkC,OAAO;QACjE1B,YAAY;IACd;IAEA,sEAAsE;IACtE,4EAA4E;IAC5E,sDAAsD;IACtDC,IAAI,CAAC,6BAA6B,GAAG,MAAMV,YACzCC,IACA,8BACA,SACAkC;IAGF,6EAA6E;IAC7EzB,IAAI,CAAC,gBAAgB,GAAG,MAAMV,YAAYC,IAAI,iBAAiB,SAASkC;IAExE,oEAAoE;IACpE,wFAAwF;IACxF,sEAAsE;IACtE,8CAA8C;IAC9C,KAAK,MAAMrI,cAAc;QACvB;QACA;QACA;QACA;QACA;QACA;QACA;QACA;QACA;QACA;QACA;QACA;QACA;QACA;QACA;QACA;QACA;QACA;QACA;QACA;QACA;QACA;QACA;QACA;QACA;QACA;QACA;QACA;KACD,CAAE;QACD4G,IAAI,CAAC5G,WAAW,GAAG,MAAMkG,YAAYC,IAAInG,YAAY,SAASqI;IAChE;IAEAzB,IAAI,CAAC,oBAAoB,GAAG,MAAMC,QAAQV,IAAI,qBAAqBkC;IACnEzB,IAAI,CAAC,sBAAsB,GAAG,MAAMV,YAClCC,IACA,uBACA,aACAkC;IAGF,2EAA2E;IAC3E,2DAA2D;IAC3D,MAAMM,UAAU,MAAMxC,GACnBnG,UAAU,CAAC,kBACXwG,KAAK,CAAC,SAAS,MAAM6B,OACrB9F,GAAG;IACN,MAAMqG,aAAiC,EAAE;IACzC,KAAK,MAAMC,UAAUF,QAAQlC,IAAI,CAAE;QACjCmC,WAAWvB,IAAI,IAAK,MAAML,YAAYb,IAAI0C,OAAO5B,GAAG,EAAE,GAAG;IAC3D;IACAL,IAAI,CAAC,iBAAiB,GAAGgC;IAEzB,yDAAyD;IACzD,MAAME,aAAa,MAAM3C,GACtBnG,UAAU,CAAC,kBACXwG,KAAK,CAAC,SAAS,MAAM6B,OACrB9F,GAAG;IACNJ,MAAM,CAAC,iBAAiB,GAAG2G,WAAWrC,IAAI,CAAC7D,MAAM;IAEjD,OAAO;QACLoD,SAAS;YAAE+B,MAAM;YAAO1B,IAAIgC;QAAM;QAClCL,aAAa,IAAIC,OAAO1C,WAAW;QACnC2C,UAAUnC,WAAW;QACrBa;QACAzE;QACAgG,WAAWrC;IACb;AACF;AAEA,2EAA2E,GAC3E,OAAO,SAASiD,eAAe5I,QAA4B;IACzD,MAAM6I,MAAM7I,SAAS6H,WAAW,CAACiB,KAAK,CAAC,GAAG;IAC1C,OAAO,CAAC,MAAM,EAAE9I,SAAS6F,OAAO,CAAC+B,IAAI,CAAC,MAAM,EAAE5H,SAAS6F,OAAO,CAACK,EAAE,CAAC,CAAC,EAAE2C,IAAI,KAAK,CAAC;AACjF;AAEA,eAAetB,eAAc"}
|
|
@@ -43,6 +43,10 @@
|
|
|
43
43
|
* the measured fan-out plus a little headroom. The process default stays at
|
|
44
44
|
* 10, so a real leak anywhere still warns, and so does a real leak on this
|
|
45
45
|
* body once it grows past the budget.
|
|
46
|
+
*
|
|
47
|
+
* `File#download()` makes the same read but never exposes the stream, so its
|
|
48
|
+
* body cannot be budgeted and it still warns. A read that wants the whole
|
|
49
|
+
* object uses {@link readStorageObject} instead.
|
|
46
50
|
*/ import { buffer } from "node:stream/consumers";
|
|
47
51
|
/**
|
|
48
52
|
* The listener limit for a Storage response body. The measured peak is 12
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../../../../../../../../libs/tenant/data/admin/src/lib/server/storage-read-stream.ts"],"sourcesContent":["/**\n * @license\n * Copyright 2026 Aglyn LLC\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n/**\n * Reading a Cloud Storage object without a false leak warning (AGL-3795).\n *\n * ## Why every Storage read used to warn\n *\n * `File#createReadStream` (and `File#download`, which is built on it) hands\n * the HTTP response body, a node-fetch `PassThrough`, to three `pipeline`s at\n * once:\n *\n * 1. node-fetch's own, from the socket into the body;\n * 2. teeny-request's, from the body into the retry-request stream, which\n * Storage only `resume()`s;\n * 3. Storage's, from the body into the read stream it returns.\n *\n * On Node 24 those add 12 `error` listeners and 11 `close` listeners to the\n * body. Node's limit is 10, so `MaxListenersExceededWarning` fired once per\n * read on every route that touches Storage. The count is a fixed fan-out: it\n * is the same for a 1-chunk and a 200-chunk object and for the first and the\n * hundredth request, and the listeners go away with the body when the read\n * ends. So nothing accumulates. The warning is a heuristic tripping on a\n * library's wiring.\n *\n * ## What this does\n *\n * Storage re-emits the body as the read stream's `response` event before it\n * or teeny-request attaches its `pipeline`. At that point, only on that one\n * per-request body, the limit is raised to {@link STORAGE_READ_BODY_LISTENER_BUDGET}:\n * the measured fan-out plus a little headroom. The process default stays at\n * 10, so a real leak anywhere still warns, and so does a real leak on this\n * body once it grows past the budget.\n */\n\nimport type { Readable } from 'node:stream'\nimport { buffer } from 'node:stream/consumers'\n\n/**\n * The listener limit for a Storage response body. The measured peak is 12\n * (`error`). Room for a few more lets a library patch add one without\n * bringing the warning back, but stays low enough that a per-chunk or\n * per-request leak would still cross it within one read.\n */\nexport const STORAGE_READ_BODY_LISTENER_BUDGET = 16\n\n/** The slice of a Storage `File` a read needs. */\nexport interface StorageReadableFile {\n createReadStream(options?: StorageReadOptions): NodeJS.ReadableStream\n}\n\n/** The `createReadStream` options the platform uses. */\nexport interface StorageReadOptions {\n start?: number\n end?: number\n validation?: 'crc32c' | 'md5' | boolean\n}\n\n/** Raises the limit on the response body Storage just received. */\nfunction budgetResponseBody(body: unknown): void {\n const emitter = body as\n | { getMaxListeners?: () => number; setMaxListeners?: (n: number) => unknown }\n | null\n | undefined\n if (typeof emitter?.setMaxListeners !== 'function') return\n const current = emitter.getMaxListeners?.() ?? 0\n // 0 means unlimited, and a higher limit was set on purpose by someone else.\n if (current === 0 || current >= STORAGE_READ_BODY_LISTENER_BUDGET) return\n emitter.setMaxListeners(STORAGE_READ_BODY_LISTENER_BUDGET)\n}\n\n/**\n * `file.createReadStream(options)`, with the response body's listener\n * limit sized to Storage's fixed fan-out (see the module note).\n */\nexport function openStorageReadStream(\n file: StorageReadableFile,\n options: StorageReadOptions = {},\n): NodeJS.ReadableStream {\n const stream = file.createReadStream(options)\n stream.once('response', budgetResponseBody)\n return stream\n}\n\n/**\n * The whole object as one Buffer, like `file.download()`, read through\n * {@link openStorageReadStream}. Rejects when the read fails.\n */\nexport async function readStorageObject(\n file: StorageReadableFile,\n options: StorageReadOptions = {},\n): Promise<Buffer> {\n return buffer(openStorageReadStream(file, options) as Readable)\n}\n"],"names":["buffer","STORAGE_READ_BODY_LISTENER_BUDGET","budgetResponseBody","body","emitter","setMaxListeners","current","getMaxListeners","openStorageReadStream","file","options","stream","createReadStream","once","readStorageObject"],"mappings":"AAAA;;;;;;;;;;;;;;;CAeC,GAED
|
|
1
|
+
{"version":3,"sources":["../../../../../../../../libs/tenant/data/admin/src/lib/server/storage-read-stream.ts"],"sourcesContent":["/**\n * @license\n * Copyright 2026 Aglyn LLC\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n/**\n * Reading a Cloud Storage object without a false leak warning (AGL-3795).\n *\n * ## Why every Storage read used to warn\n *\n * `File#createReadStream` (and `File#download`, which is built on it) hands\n * the HTTP response body, a node-fetch `PassThrough`, to three `pipeline`s at\n * once:\n *\n * 1. node-fetch's own, from the socket into the body;\n * 2. teeny-request's, from the body into the retry-request stream, which\n * Storage only `resume()`s;\n * 3. Storage's, from the body into the read stream it returns.\n *\n * On Node 24 those add 12 `error` listeners and 11 `close` listeners to the\n * body. Node's limit is 10, so `MaxListenersExceededWarning` fired once per\n * read on every route that touches Storage. The count is a fixed fan-out: it\n * is the same for a 1-chunk and a 200-chunk object and for the first and the\n * hundredth request, and the listeners go away with the body when the read\n * ends. So nothing accumulates. The warning is a heuristic tripping on a\n * library's wiring.\n *\n * ## What this does\n *\n * Storage re-emits the body as the read stream's `response` event before it\n * or teeny-request attaches its `pipeline`. At that point, only on that one\n * per-request body, the limit is raised to {@link STORAGE_READ_BODY_LISTENER_BUDGET}:\n * the measured fan-out plus a little headroom. The process default stays at\n * 10, so a real leak anywhere still warns, and so does a real leak on this\n * body once it grows past the budget.\n *\n * `File#download()` makes the same read but never exposes the stream, so its\n * body cannot be budgeted and it still warns. A read that wants the whole\n * object uses {@link readStorageObject} instead.\n */\n\nimport type { Readable } from 'node:stream'\nimport { buffer } from 'node:stream/consumers'\n\n/**\n * The listener limit for a Storage response body. The measured peak is 12\n * (`error`). Room for a few more lets a library patch add one without\n * bringing the warning back, but stays low enough that a per-chunk or\n * per-request leak would still cross it within one read.\n */\nexport const STORAGE_READ_BODY_LISTENER_BUDGET = 16\n\n/** The slice of a Storage `File` a read needs. */\nexport interface StorageReadableFile {\n createReadStream(options?: StorageReadOptions): NodeJS.ReadableStream\n}\n\n/** The `createReadStream` options the platform uses. */\nexport interface StorageReadOptions {\n start?: number\n end?: number\n validation?: 'crc32c' | 'md5' | boolean\n}\n\n/** Raises the limit on the response body Storage just received. */\nfunction budgetResponseBody(body: unknown): void {\n const emitter = body as\n | { getMaxListeners?: () => number; setMaxListeners?: (n: number) => unknown }\n | null\n | undefined\n if (typeof emitter?.setMaxListeners !== 'function') return\n const current = emitter.getMaxListeners?.() ?? 0\n // 0 means unlimited, and a higher limit was set on purpose by someone else.\n if (current === 0 || current >= STORAGE_READ_BODY_LISTENER_BUDGET) return\n emitter.setMaxListeners(STORAGE_READ_BODY_LISTENER_BUDGET)\n}\n\n/**\n * `file.createReadStream(options)`, with the response body's listener\n * limit sized to Storage's fixed fan-out (see the module note).\n */\nexport function openStorageReadStream(\n file: StorageReadableFile,\n options: StorageReadOptions = {},\n): NodeJS.ReadableStream {\n const stream = file.createReadStream(options)\n stream.once('response', budgetResponseBody)\n return stream\n}\n\n/**\n * The whole object as one Buffer, like `file.download()`, read through\n * {@link openStorageReadStream}. Rejects when the read fails.\n */\nexport async function readStorageObject(\n file: StorageReadableFile,\n options: StorageReadOptions = {},\n): Promise<Buffer> {\n return buffer(openStorageReadStream(file, options) as Readable)\n}\n"],"names":["buffer","STORAGE_READ_BODY_LISTENER_BUDGET","budgetResponseBody","body","emitter","setMaxListeners","current","getMaxListeners","openStorageReadStream","file","options","stream","createReadStream","once","readStorageObject"],"mappings":"AAAA;;;;;;;;;;;;;;;CAeC,GAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAkCC,GAGD,SAASA,MAAM,QAAQ,wBAAuB;AAE9C;;;;;CAKC,GACD,OAAO,MAAMC,oCAAoC,GAAE;AAcnD,iEAAiE,GACjE,SAASC,mBAAmBC,IAAa;;IACvC,MAAMC,UAAUD;IAIhB,IAAI,QAAOC,2BAAAA,QAASC,eAAe,MAAK,YAAY;IACpD,MAAMC,kBAAUF,QAAQG,eAAe,oBAAvBH,QAAQG,eAAe,MAAvBH,2BAA+B;IAC/C,4EAA4E;IAC5E,IAAIE,YAAY,KAAKA,WAAWL,mCAAmC;IACnEG,QAAQC,eAAe,CAACJ;AAC1B;AAEA;;;CAGC,GACD,OAAO,SAASO,sBACdC,IAAyB,EACzBC,UAA8B,CAAC,CAAC;IAEhC,MAAMC,SAASF,KAAKG,gBAAgB,CAACF;IACrCC,OAAOE,IAAI,CAAC,YAAYX;IACxB,OAAOS;AACT;AAEA;;;CAGC,GACD,OAAO,eAAeG,kBACpBL,IAAyB,EACzBC,UAA8B,CAAC,CAAC;IAEhC,OAAOV,OAAOQ,sBAAsBC,MAAMC;AAC5C"}
|