@superbee/core 0.2.0-pre.30 → 0.2.0-pre.32
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -0
- package/dist/bundle-health/report.d.ts.map +1 -1
- package/dist/bundle-health/report.js +13 -1
- package/dist/bundle-health/report.js.map +1 -1
- package/dist/connectors.d.ts +10 -0
- package/dist/connectors.d.ts.map +1 -1
- package/dist/connectors.js +13 -2
- package/dist/connectors.js.map +1 -1
- package/dist/link-types.d.ts +3 -2
- package/dist/link-types.d.ts.map +1 -1
- package/dist/link-types.js +6 -3
- package/dist/link-types.js.map +1 -1
- package/dist/recipe-set-preflight.d.ts +139 -0
- package/dist/recipe-set-preflight.d.ts.map +1 -0
- package/dist/recipe-set-preflight.js +536 -0
- package/dist/recipe-set-preflight.js.map +1 -0
- package/dist/record-overlay/index.d.ts +1 -1
- package/dist/record-overlay/index.d.ts.map +1 -1
- package/dist/record-overlay/index.js +1 -1
- package/dist/record-overlay/index.js.map +1 -1
- package/dist/record-overlay/policy.d.ts +5 -0
- package/dist/record-overlay/policy.d.ts.map +1 -1
- package/dist/record-overlay/policy.js +10 -3
- package/dist/record-overlay/policy.js.map +1 -1
- package/dist/source-engine/driver.d.ts +40 -0
- package/dist/source-engine/driver.d.ts.map +1 -0
- package/dist/source-engine/driver.js +392 -0
- package/dist/source-engine/driver.js.map +1 -0
- package/dist/source-engine/evidence.d.ts +39 -0
- package/dist/source-engine/evidence.d.ts.map +1 -0
- package/dist/source-engine/evidence.js +54 -0
- package/dist/source-engine/evidence.js.map +1 -0
- package/dist/source-engine/index.d.ts +7 -0
- package/dist/source-engine/index.d.ts.map +1 -1
- package/dist/source-engine/index.js +5 -0
- package/dist/source-engine/index.js.map +1 -1
- package/dist/source-engine/observe.d.ts +1 -9
- package/dist/source-engine/observe.d.ts.map +1 -1
- package/dist/source-engine/observe.js +14 -374
- package/dist/source-engine/observe.js.map +1 -1
- package/dist/source-engine/overlay.d.ts +2 -1
- package/dist/source-engine/overlay.d.ts.map +1 -1
- package/dist/source-engine/overlay.js +3 -3
- package/dist/source-engine/overlay.js.map +1 -1
- package/dist/source-engine/snapshot.d.ts +104 -0
- package/dist/source-engine/snapshot.d.ts.map +1 -0
- package/dist/source-engine/snapshot.js +240 -0
- package/dist/source-engine/snapshot.js.map +1 -0
- package/dist/source-profiles/index.d.ts +1 -1
- package/dist/source-profiles/index.js +1 -1
- package/package.json +5 -1
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
/** Caller-associated applicability of original input bytes; independent of document freshness. */
|
|
2
|
+
export interface SnapshotEvidenceIdentity {
|
|
3
|
+
id: string;
|
|
4
|
+
sha256: string;
|
|
5
|
+
}
|
|
6
|
+
export interface SnapshotEvidence {
|
|
7
|
+
v: 1;
|
|
8
|
+
association: "caller_asserted";
|
|
9
|
+
inputs: SnapshotEvidenceIdentity[];
|
|
10
|
+
settings_digest?: string;
|
|
11
|
+
required_identities?: SnapshotEvidenceIdentity[];
|
|
12
|
+
}
|
|
13
|
+
export interface SnapshotEvidenceObservation {
|
|
14
|
+
inputs: Readonly<Record<string, string | null | undefined>>;
|
|
15
|
+
required_identities?: Readonly<Record<string, string | null | undefined>>;
|
|
16
|
+
settings_digest?: string | null;
|
|
17
|
+
}
|
|
18
|
+
export interface SnapshotFreshnessRow {
|
|
19
|
+
dimension: "input" | "required_identity" | "settings";
|
|
20
|
+
id: string;
|
|
21
|
+
status: "equal" | "mismatch" | "unknown";
|
|
22
|
+
}
|
|
23
|
+
export interface SnapshotFreshness {
|
|
24
|
+
status: "current" | "stale" | "unverifiable";
|
|
25
|
+
association: "caller_asserted" | null;
|
|
26
|
+
basis: "declared_original_inputs";
|
|
27
|
+
reason: "all_required_equal" | "known_mismatch" | "missing_dimension" | "no_input_basis";
|
|
28
|
+
rows: SnapshotFreshnessRow[];
|
|
29
|
+
}
|
|
30
|
+
export declare const SNAPSHOT_SHA256: RegExp;
|
|
31
|
+
export declare class SnapshotEvidenceRefusal extends Error {
|
|
32
|
+
readonly code = "SNAPSHOT_EVIDENCE_INVALID";
|
|
33
|
+
constructor();
|
|
34
|
+
}
|
|
35
|
+
/** Closed, bounded portable evidence. Logical IDs select no file or executable. */
|
|
36
|
+
export declare function parseSnapshotEvidence(value: unknown): SnapshotEvidence;
|
|
37
|
+
/** Known mismatches take precedence over unknown dimensions, including an empty input basis. */
|
|
38
|
+
export declare function assessSnapshotFreshness(expected: SnapshotEvidence | undefined, observed: SnapshotEvidenceObservation): SnapshotFreshness;
|
|
39
|
+
//# sourceMappingURL=evidence.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"evidence.d.ts","sourceRoot":"","sources":["../../src/source-engine/evidence.ts"],"names":[],"mappings":"AAAA,kGAAkG;AAClG,MAAM,WAAW,wBAAwB;IAAG,EAAE,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE;AACxE,MAAM,WAAW,gBAAgB;IAC/B,CAAC,EAAE,CAAC,CAAC;IACL,WAAW,EAAE,iBAAiB,CAAC;IAC/B,MAAM,EAAE,wBAAwB,EAAE,CAAC;IACnC,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,mBAAmB,CAAC,EAAE,wBAAwB,EAAE,CAAC;CAClD;AACD,MAAM,WAAW,2BAA2B;IAC1C,MAAM,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC,CAAC,CAAC;IAC5D,mBAAmB,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC,CAAC,CAAC;IAC1E,eAAe,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CACjC;AACD,MAAM,WAAW,oBAAoB;IACnC,SAAS,EAAE,OAAO,GAAG,mBAAmB,GAAG,UAAU,CAAC;IACtD,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,OAAO,GAAG,UAAU,GAAG,SAAS,CAAC;CAC1C;AACD,MAAM,WAAW,iBAAiB;IAChC,MAAM,EAAE,SAAS,GAAG,OAAO,GAAG,cAAc,CAAC;IAC7C,WAAW,EAAE,iBAAiB,GAAG,IAAI,CAAC;IACtC,KAAK,EAAE,0BAA0B,CAAC;IAClC,MAAM,EAAE,oBAAoB,GAAG,gBAAgB,GAAG,mBAAmB,GAAG,gBAAgB,CAAC;IACzF,IAAI,EAAE,oBAAoB,EAAE,CAAC;CAC9B;AACD,eAAO,MAAM,eAAe,QAA0B,CAAC;AAKvD,qBAAa,uBAAwB,SAAQ,KAAK;IAChD,QAAQ,CAAC,IAAI,+BAA+B;;CAE7C;AACD,mFAAmF;AACnF,wBAAgB,qBAAqB,CAAC,KAAK,EAAE,OAAO,GAAG,gBAAgB,CAmBtE;AAED,gGAAgG;AAChG,wBAAgB,uBAAuB,CAAC,QAAQ,EAAE,gBAAgB,GAAG,SAAS,EAAE,QAAQ,EAAE,2BAA2B,GAAG,iBAAiB,CAgBxI"}
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
export const SNAPSHOT_SHA256 = /^sha256:[0-9a-f]{64}$/;
|
|
2
|
+
const LABEL = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/;
|
|
3
|
+
const object = (v) => v !== null && typeof v === "object" && !Array.isArray(v);
|
|
4
|
+
const closed = (v, keys) => Object.keys(v).every(k => keys.includes(k));
|
|
5
|
+
export class SnapshotEvidenceRefusal extends Error {
|
|
6
|
+
code = "SNAPSHOT_EVIDENCE_INVALID";
|
|
7
|
+
constructor() { super("snapshot evidence is not a closed v1 caller-associated descriptor"); this.name = "SnapshotEvidenceRefusal"; }
|
|
8
|
+
}
|
|
9
|
+
/** Closed, bounded portable evidence. Logical IDs select no file or executable. */
|
|
10
|
+
export function parseSnapshotEvidence(value) {
|
|
11
|
+
const refuse = () => { throw new SnapshotEvidenceRefusal(); };
|
|
12
|
+
if (!object(value) || !closed(value, ["v", "association", "inputs", "settings_digest", "required_identities"])
|
|
13
|
+
|| value.v !== 1 || value.association !== "caller_asserted")
|
|
14
|
+
return refuse();
|
|
15
|
+
const identities = (raw) => {
|
|
16
|
+
if (!Array.isArray(raw) || raw.length > 10_000)
|
|
17
|
+
return refuse();
|
|
18
|
+
const seen = new Set();
|
|
19
|
+
return raw.map(row => {
|
|
20
|
+
if (!object(row) || !closed(row, ["id", "sha256"]) || typeof row.id !== "string" || !LABEL.test(row.id)
|
|
21
|
+
|| typeof row.sha256 !== "string" || !SNAPSHOT_SHA256.test(row.sha256) || seen.has(row.id))
|
|
22
|
+
return refuse();
|
|
23
|
+
seen.add(row.id);
|
|
24
|
+
return { id: row.id, sha256: row.sha256 };
|
|
25
|
+
});
|
|
26
|
+
};
|
|
27
|
+
const inputs = identities(value.inputs);
|
|
28
|
+
if (value.settings_digest !== undefined && (typeof value.settings_digest !== "string" || !SNAPSHOT_SHA256.test(value.settings_digest)))
|
|
29
|
+
return refuse();
|
|
30
|
+
return { v: 1, association: "caller_asserted", inputs,
|
|
31
|
+
...(value.settings_digest !== undefined ? { settings_digest: value.settings_digest } : {}),
|
|
32
|
+
...(value.required_identities !== undefined ? { required_identities: identities(value.required_identities) } : {}) };
|
|
33
|
+
}
|
|
34
|
+
/** Known mismatches take precedence over unknown dimensions, including an empty input basis. */
|
|
35
|
+
export function assessSnapshotFreshness(expected, observed) {
|
|
36
|
+
const evidence = expected === undefined ? undefined : parseSnapshotEvidence(expected);
|
|
37
|
+
const rows = [];
|
|
38
|
+
const row = (dimension, id, wanted, actual) => {
|
|
39
|
+
rows.push({ dimension, id, status: typeof actual !== "string" || !SNAPSHOT_SHA256.test(actual) ? "unknown" : actual === wanted ? "equal" : "mismatch" });
|
|
40
|
+
};
|
|
41
|
+
for (const input of evidence?.inputs ?? [])
|
|
42
|
+
row("input", input.id, input.sha256, Object.hasOwn(observed.inputs, input.id) ? observed.inputs[input.id] : undefined);
|
|
43
|
+
for (const identity of evidence?.required_identities ?? [])
|
|
44
|
+
row("required_identity", identity.id, identity.sha256, observed.required_identities && Object.hasOwn(observed.required_identities, identity.id) ? observed.required_identities[identity.id] : undefined);
|
|
45
|
+
if (evidence?.settings_digest !== undefined)
|
|
46
|
+
row("settings", "original_settings", evidence.settings_digest, observed.settings_digest);
|
|
47
|
+
const mismatch = rows.some(r => r.status === "mismatch");
|
|
48
|
+
const basis = (evidence?.inputs.length ?? 0) > 0;
|
|
49
|
+
const unknown = rows.some(r => r.status === "unknown");
|
|
50
|
+
return { status: mismatch ? "stale" : !basis || unknown ? "unverifiable" : "current",
|
|
51
|
+
association: evidence?.association ?? null, basis: "declared_original_inputs",
|
|
52
|
+
reason: mismatch ? "known_mismatch" : !basis ? "no_input_basis" : unknown ? "missing_dimension" : "all_required_equal", rows };
|
|
53
|
+
}
|
|
54
|
+
//# sourceMappingURL=evidence.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"evidence.js","sourceRoot":"","sources":["../../src/source-engine/evidence.ts"],"names":[],"mappings":"AA0BA,MAAM,CAAC,MAAM,eAAe,GAAG,uBAAuB,CAAC;AACvD,MAAM,KAAK,GAAG,qCAAqC,CAAC;AACpD,MAAM,MAAM,GAAG,CAAC,CAAU,EAAgC,EAAE,CAAC,CAAC,KAAK,IAAI,IAAI,OAAO,CAAC,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;AACtH,MAAM,MAAM,GAAG,CAAC,CAA0B,EAAE,IAAuB,EAAW,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;AAE7H,MAAM,OAAO,uBAAwB,SAAQ,KAAK;IACvC,IAAI,GAAG,2BAA2B,CAAC;IAC5C,gBAAgB,KAAK,CAAC,mEAAmE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,GAAG,yBAAyB,CAAC,CAAC,CAAC;CACrI;AACD,mFAAmF;AACnF,MAAM,UAAU,qBAAqB,CAAC,KAAc;IAClD,MAAM,MAAM,GAAG,GAAU,EAAE,GAAG,MAAM,IAAI,uBAAuB,EAAE,CAAC,CAAC,CAAC,CAAC;IACrE,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC,GAAG,EAAE,aAAa,EAAE,QAAQ,EAAE,iBAAiB,EAAE,qBAAqB,CAAC,CAAC;WACzG,KAAK,CAAC,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,WAAW,KAAK,iBAAiB;QAAE,OAAO,MAAM,EAAE,CAAC;IAC/E,MAAM,UAAU,GAAG,CAAC,GAAY,EAA8B,EAAE;QAC9D,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,GAAG,CAAC,MAAM,GAAG,MAAM;YAAE,OAAO,MAAM,EAAE,CAAC;QAChE,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;QAC/B,OAAO,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE;YACnB,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC,IAAI,OAAO,GAAG,CAAC,EAAE,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;mBAClG,OAAO,GAAG,CAAC,MAAM,KAAK,QAAQ,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;gBAAE,OAAO,MAAM,EAAE,CAAC;YAC9G,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YACjB,OAAO,EAAE,EAAE,EAAE,GAAG,CAAC,EAAE,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,CAAC;QAC5C,CAAC,CAAC,CAAC;IACL,CAAC,CAAC;IACF,MAAM,MAAM,GAAG,UAAU,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;IACxC,IAAI,KAAK,CAAC,eAAe,KAAK,SAAS,IAAI,CAAC,OAAO,KAAK,CAAC,eAAe,KAAK,QAAQ,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC;QAAE,OAAO,MAAM,EAAE,CAAC;IACxJ,OAAO,EAAE,CAAC,EAAE,CAAC,EAAE,WAAW,EAAE,iBAAiB,EAAE,MAAM;QACnD,GAAG,CAAC,KAAK,CAAC,eAAe,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,eAAe,EAAE,KAAK,CAAC,eAAyB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACpG,GAAG,CAAC,KAAK,CAAC,mBAAmB,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,mBAAmB,EAAE,UAAU,CAAC,KAAK,CAAC,mBAAmB,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC;AACzH,CAAC;AAED,gGAAgG;AAChG,MAAM,UAAU,uBAAuB,CAAC,QAAsC,EAAE,QAAqC;IACnH,MAAM,QAAQ,GAAG,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,qBAAqB,CAAC,QAAQ,CAAC,CAAC;IACtF,MAAM,IAAI,GAA2B,EAAE,CAAC;IACxC,MAAM,GAAG,GAAG,CAAC,SAA4C,EAAE,EAAU,EAAE,MAAc,EAAE,MAAe,EAAQ,EAAE;QAC9G,IAAI,CAAC,IAAI,CAAC,EAAE,SAAS,EAAE,EAAE,EAAE,MAAM,EAAE,OAAO,MAAM,KAAK,QAAQ,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM,KAAK,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,UAAU,EAAE,CAAC,CAAC;IAC3J,CAAC,CAAC;IACF,KAAK,MAAM,KAAK,IAAI,QAAQ,EAAE,MAAM,IAAI,EAAE;QAAE,GAAG,CAAC,OAAO,EAAE,KAAK,CAAC,EAAE,EAAE,KAAK,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,EAAE,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;IACnK,KAAK,MAAM,QAAQ,IAAI,QAAQ,EAAE,mBAAmB,IAAI,EAAE;QAAE,GAAG,CAAC,mBAAmB,EAAE,QAAQ,CAAC,EAAE,EAAE,QAAQ,CAAC,MAAM,EAC/G,QAAQ,CAAC,mBAAmB,IAAI,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,mBAAmB,EAAE,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,mBAAmB,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;IACpJ,IAAI,QAAQ,EAAE,eAAe,KAAK,SAAS;QAAE,GAAG,CAAC,UAAU,EAAE,mBAAmB,EAAE,QAAQ,CAAC,eAAe,EAAE,QAAQ,CAAC,eAAe,CAAC,CAAC;IACtI,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,UAAU,CAAC,CAAC;IACzD,MAAM,KAAK,GAAG,CAAC,QAAQ,EAAE,MAAM,CAAC,MAAM,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;IACjD,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC;IACvD,OAAO,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,KAAK,IAAI,OAAO,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,SAAS;QAClF,WAAW,EAAE,QAAQ,EAAE,WAAW,IAAI,IAAI,EAAE,KAAK,EAAE,0BAA0B;QAC7E,MAAM,EAAE,QAAQ,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,mBAAmB,CAAC,CAAC,CAAC,oBAAoB,EAAE,IAAI,EAAE,CAAC;AACnI,CAAC"}
|
|
@@ -11,6 +11,9 @@
|
|
|
11
11
|
* the cursor rule (a cursor is stored only after its page commits), completeness, and the diff into
|
|
12
12
|
* record-overlay changes, with disappearance only from a complete observation.
|
|
13
13
|
*
|
|
14
|
+
* Finite local inputs use bindSnapshotSource/decodeSnapshot/admitSnapshot/observeSnapshot;
|
|
15
|
+
* admission precedes reconciliation and the same internal driver owns state transitions.
|
|
16
|
+
*
|
|
14
17
|
* Storage-neutral and Worker-free: no Node globals or builtins, and nothing here calls `fetch`.
|
|
15
18
|
*/
|
|
16
19
|
export { canonicalRobotsPath, fetchIdentity, parseRobots, refusedAddress, robotsDecision, robotsRuleMatches, } from "./robots.js";
|
|
@@ -23,4 +26,8 @@ export type { MachineState, PendingChange, RecordEntry, SourceState } from "./ma
|
|
|
23
26
|
export type { CommitResult, HttpMethod, NormalizedRecord, ObserveContext, ObservePage, ObserveSourceOptions, PageCommit, ProfileDiagnostic, ProfileRequest, ProfileResponse, ProfileTransport, RequestRules, SourceBudget, SourceOutcome, SourceProfile, SourceRequestRow, SourceRunReceipt, SpecResult, StatusRules, Transport, TransportRequest, TransportResult, } from "./types.js";
|
|
24
27
|
export { createSourceOverlayAdapter, isSourceOverlaySnapshot, MAX_SOURCE_OVERLAY_ENTRIES } from "./overlay.js";
|
|
25
28
|
export type { SourceOverlayAdapter, SourceOverlayEntry, SourceOverlayOptions, SourceOverlayResult, SourceOverlaySnapshot } from "./overlay.js";
|
|
29
|
+
export { SNAPSHOT_MAX_BYTES, SNAPSHOT_MAX_RECORDS, SNAPSHOT_PROFILE, SNAPSHOT_REFUSAL_CODES, SnapshotRefusal, bindSnapshotSource, decodeSnapshot, admitSnapshot, observeSnapshot } from "./snapshot.js";
|
|
30
|
+
export type { BoundSnapshotSource, SnapshotDocument, DecodedSnapshot, SnapshotRefusalCode, SnapshotCapacityInput, SnapshotCapacityFacts, SnapshotAdmissionOptions, SnapshotAdmission, ObserveSnapshotOptions, SnapshotRunReceipt } from "./snapshot.js";
|
|
31
|
+
export { SnapshotEvidenceRefusal, parseSnapshotEvidence, assessSnapshotFreshness } from "./evidence.js";
|
|
32
|
+
export type { SnapshotEvidenceIdentity, SnapshotEvidence, SnapshotEvidenceObservation, SnapshotFreshnessRow, SnapshotFreshness } from "./evidence.js";
|
|
26
33
|
//# sourceMappingURL=index.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/source-engine/index.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/source-engine/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AACH,OAAO,EACL,mBAAmB,EACnB,aAAa,EACb,WAAW,EACX,cAAc,EACd,cAAc,EACd,iBAAiB,GAClB,MAAM,aAAa,CAAC;AACrB,YAAY,EAAE,aAAa,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAC9D,OAAO,EAAE,qBAAqB,EAAE,oBAAoB,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AACxF,YAAY,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAC1C,OAAO,EAAE,yBAAyB,EAAE,gBAAgB,EAAE,mBAAmB,EAAE,aAAa,EAAE,kBAAkB,EAAE,MAAM,cAAc,CAAC;AACnI,OAAO,EAAE,aAAa,EAAE,oBAAoB,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AAC7F,YAAY,EAAE,YAAY,EAAE,aAAa,EAAE,WAAW,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAChG,YAAY,EACV,YAAY,EACZ,UAAU,EACV,gBAAgB,EAChB,cAAc,EACd,WAAW,EACX,oBAAoB,EACpB,UAAU,EACV,iBAAiB,EACjB,cAAc,EACd,eAAe,EACf,gBAAgB,EAChB,YAAY,EACZ,YAAY,EACZ,aAAa,EACb,aAAa,EACb,gBAAgB,EAChB,gBAAgB,EAChB,UAAU,EACV,WAAW,EACX,SAAS,EACT,gBAAgB,EAChB,eAAe,GAChB,MAAM,YAAY,CAAC;AACpB,OAAO,EAAE,0BAA0B,EAAE,uBAAuB,EAAE,0BAA0B,EAAE,MAAM,cAAc,CAAC;AAC/G,YAAY,EAAE,oBAAoB,EAAE,kBAAkB,EAAE,oBAAoB,EAAE,mBAAmB,EAAE,qBAAqB,EAAE,MAAM,cAAc,CAAC;AAC/I,OAAO,EAAE,kBAAkB,EAAE,oBAAoB,EAAE,gBAAgB,EAAE,sBAAsB,EAAE,eAAe,EAC1G,kBAAkB,EAAE,cAAc,EAAE,aAAa,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AAC5F,YAAY,EAAE,mBAAmB,EAAE,gBAAgB,EAAE,eAAe,EAAE,mBAAmB,EAAE,qBAAqB,EAAE,qBAAqB,EACrI,wBAAwB,EAAE,iBAAiB,EAAE,sBAAsB,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAC;AACjH,OAAO,EAAE,uBAAuB,EAAE,qBAAqB,EAAE,uBAAuB,EAAE,MAAM,eAAe,CAAC;AACxG,YAAY,EAAE,wBAAwB,EAAE,gBAAgB,EAAE,2BAA2B,EAAE,oBAAoB,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC"}
|
|
@@ -11,6 +11,9 @@
|
|
|
11
11
|
* the cursor rule (a cursor is stored only after its page commits), completeness, and the diff into
|
|
12
12
|
* record-overlay changes, with disappearance only from a complete observation.
|
|
13
13
|
*
|
|
14
|
+
* Finite local inputs use bindSnapshotSource/decodeSnapshot/admitSnapshot/observeSnapshot;
|
|
15
|
+
* admission precedes reconciliation and the same internal driver owns state transitions.
|
|
16
|
+
*
|
|
14
17
|
* Storage-neutral and Worker-free: no Node globals or builtins, and nothing here calls `fetch`.
|
|
15
18
|
*/
|
|
16
19
|
export { canonicalRobotsPath, fetchIdentity, parseRobots, refusedAddress, robotsDecision, robotsRuleMatches, } from "./robots.js";
|
|
@@ -18,4 +21,6 @@ export { DEFAULT_SOURCE_BUDGET, DEFAULT_STATUS_RULES, sourceHostSet } from "./gu
|
|
|
18
21
|
export { DEFAULT_MISSING_THRESHOLD, MAX_CURSOR_CHARS, MAX_REPLAY_FAILURES, observeSource, unsupportedContent } from "./observe.js";
|
|
19
22
|
export { isSourceState, MachineStateConflict, MemoryMachineState } from "./machine-state.js";
|
|
20
23
|
export { createSourceOverlayAdapter, isSourceOverlaySnapshot, MAX_SOURCE_OVERLAY_ENTRIES } from "./overlay.js";
|
|
24
|
+
export { SNAPSHOT_MAX_BYTES, SNAPSHOT_MAX_RECORDS, SNAPSHOT_PROFILE, SNAPSHOT_REFUSAL_CODES, SnapshotRefusal, bindSnapshotSource, decodeSnapshot, admitSnapshot, observeSnapshot } from "./snapshot.js";
|
|
25
|
+
export { SnapshotEvidenceRefusal, parseSnapshotEvidence, assessSnapshotFreshness } from "./evidence.js";
|
|
21
26
|
//# sourceMappingURL=index.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/source-engine/index.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/source-engine/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AACH,OAAO,EACL,mBAAmB,EACnB,aAAa,EACb,WAAW,EACX,cAAc,EACd,cAAc,EACd,iBAAiB,GAClB,MAAM,aAAa,CAAC;AAErB,OAAO,EAAE,qBAAqB,EAAE,oBAAoB,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAExF,OAAO,EAAE,yBAAyB,EAAE,gBAAgB,EAAE,mBAAmB,EAAE,aAAa,EAAE,kBAAkB,EAAE,MAAM,cAAc,CAAC;AACnI,OAAO,EAAE,aAAa,EAAE,oBAAoB,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AA0B7F,OAAO,EAAE,0BAA0B,EAAE,uBAAuB,EAAE,0BAA0B,EAAE,MAAM,cAAc,CAAC;AAE/G,OAAO,EAAE,kBAAkB,EAAE,oBAAoB,EAAE,gBAAgB,EAAE,sBAAsB,EAAE,eAAe,EAC1G,kBAAkB,EAAE,cAAc,EAAE,aAAa,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AAG5F,OAAO,EAAE,uBAAuB,EAAE,qBAAqB,EAAE,uBAAuB,EAAE,MAAM,eAAe,CAAC"}
|
|
@@ -1,13 +1,5 @@
|
|
|
1
1
|
import type { ObserveSourceOptions, SourceRunReceipt } from "./types.js";
|
|
2
|
-
|
|
3
|
-
export declare const DEFAULT_MISSING_THRESHOLD = 2;
|
|
4
|
-
/** The largest stored cursor, as canonical JSON characters. */
|
|
5
|
-
export declare const MAX_CURSOR_CHARS = 8192;
|
|
6
|
-
/**
|
|
7
|
-
* Replays of a pending change whose commit threw, after which the change is dropped with a finding:
|
|
8
|
-
* the record is planned again when it is next observed.
|
|
9
|
-
*/
|
|
10
|
-
export declare const MAX_REPLAY_FAILURES = 5;
|
|
2
|
+
export { DEFAULT_MISSING_THRESHOLD, MAX_CURSOR_CHARS, MAX_REPLAY_FAILURES } from "./driver.js";
|
|
11
3
|
/** Thrown by a profile that cannot read a provider's body; the run ends `unsupported_content`. */
|
|
12
4
|
export declare function unsupportedContent(detail: string): never;
|
|
13
5
|
/**
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"observe.d.ts","sourceRoot":"","sources":["../../src/source-engine/observe.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"observe.d.ts","sourceRoot":"","sources":["../../src/source-engine/observe.ts"],"names":[],"mappings":"AAeA,OAAO,KAAK,EACV,oBAAoB,EAGpB,gBAAgB,EAEjB,MAAM,YAAY,CAAC;AAEpB,OAAO,EAAE,yBAAyB,EAAE,gBAAgB,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAC;AAM/F,kGAAkG;AAClG,wBAAgB,kBAAkB,CAAC,MAAM,EAAE,MAAM,GAAG,KAAK,CAExD;AAoED;;;GAGG;AACH,wBAAsB,aAAa,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,oBAAoB,CAAC,IAAI,EAAE,MAAM,CAAC,GAAG,OAAO,CAAC,gBAAgB,CAAC,CAsJxH"}
|
|
@@ -9,45 +9,18 @@ import { validateConnectorRequest, declarationRequestRules, placeholders, provid
|
|
|
9
9
|
* source is declared complete; only a complete observation counts a stored record as missing, and
|
|
10
10
|
* a record is `disappeared` after `missing_threshold` of them.
|
|
11
11
|
*/
|
|
12
|
-
import {
|
|
13
|
-
import {
|
|
14
|
-
import { DEFAULT_SOURCE_BUDGET, DEFAULT_STATUS_RULES, guardTransport, raceAbort, sourceHostSet, SourceStop } from "./guard.js";
|
|
12
|
+
import { findingsFor } from "../connectors.js";
|
|
13
|
+
import { DEFAULT_SOURCE_BUDGET, DEFAULT_STATUS_RULES, guardTransport, raceAbort, sourceHostSet } from "./guard.js";
|
|
15
14
|
import { isSourceState } from "./machine-state.js";
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
/** The largest stored cursor, as canonical JSON characters. */
|
|
19
|
-
export const MAX_CURSOR_CHARS = 8192;
|
|
20
|
-
/**
|
|
21
|
-
* Replays of a pending change whose commit threw, after which the change is dropped with a finding:
|
|
22
|
-
* the record is planned again when it is next observed.
|
|
23
|
-
*/
|
|
24
|
-
export const MAX_REPLAY_FAILURES = 5;
|
|
15
|
+
export { DEFAULT_MISSING_THRESHOLD, MAX_CURSOR_CHARS, MAX_REPLAY_FAILURES } from "./driver.js";
|
|
16
|
+
import { driveObservation, UNSUPPORTED_CONTENT } from "./driver.js";
|
|
25
17
|
const PROFILE_ID = /^[a-z][a-z0-9-]*$/;
|
|
26
|
-
const UNSUPPORTED_CONTENT = "SOURCE_UNSUPPORTED_CONTENT";
|
|
27
18
|
/** Thrown by a profile that cannot read a provider's body; the run ends `unsupported_content`. */
|
|
28
19
|
export function unsupportedContent(detail) {
|
|
29
20
|
throw Object.assign(new Error(`unsupported content: ${detail}`), { code: UNSUPPORTED_CONTENT, detail });
|
|
30
21
|
}
|
|
31
22
|
const record = (value) => value !== null && typeof value === "object" && !Array.isArray(value);
|
|
32
|
-
const clip = (text, n) =>
|
|
33
|
-
/** JSON data only: finite numbers, plain objects and arrays, no undefined members. */
|
|
34
|
-
function isJsonData(value, depth = 0) {
|
|
35
|
-
if (depth > 32)
|
|
36
|
-
return false;
|
|
37
|
-
if (value === null || typeof value === "string" || typeof value === "boolean")
|
|
38
|
-
return true;
|
|
39
|
-
if (typeof value === "number")
|
|
40
|
-
return Number.isFinite(value);
|
|
41
|
-
if (Array.isArray(value))
|
|
42
|
-
return value.every((item) => isJsonData(item, depth + 1));
|
|
43
|
-
if (!record(value))
|
|
44
|
-
return false;
|
|
45
|
-
// A plain object from any realm: its prototype is null or a prototype that itself has none.
|
|
46
|
-
const proto = Object.getPrototypeOf(value);
|
|
47
|
-
if (proto !== null && Object.getPrototypeOf(proto) !== null)
|
|
48
|
-
return false;
|
|
49
|
-
return Object.values(value).every((item) => item !== undefined && isJsonData(item, depth + 1));
|
|
50
|
-
}
|
|
23
|
+
const clip = (text, n) => text.length > n ? text.slice(0, n - 1) + "…" : text;
|
|
51
24
|
function budgetOf(given) {
|
|
52
25
|
const budget = { ...DEFAULT_SOURCE_BUDGET, ...given };
|
|
53
26
|
// Timer delays above 2^31 - 1 milliseconds overflow to an immediate timeout.
|
|
@@ -73,96 +46,6 @@ function profileProblem(profile) {
|
|
|
73
46
|
return "missing a method";
|
|
74
47
|
return undefined;
|
|
75
48
|
}
|
|
76
|
-
/** The page as the engine accepts it, or why not. */
|
|
77
|
-
function pageProblem(page) {
|
|
78
|
-
if (!record(page))
|
|
79
|
-
return "page_invalid";
|
|
80
|
-
if (!Array.isArray(page.records) || typeof page.more !== "boolean" || typeof page.complete !== "boolean")
|
|
81
|
-
return "page_invalid";
|
|
82
|
-
if (page.diagnostics !== undefined && !(Array.isArray(page.diagnostics) && page.diagnostics.every((d) => record(d) && typeof d.code === "string" && typeof d.message === "string")))
|
|
83
|
-
return "page_invalid";
|
|
84
|
-
if (page.validators !== undefined && !(record(page.validators) && Object.values(page.validators).every((v) => typeof v === "string")))
|
|
85
|
-
return "page_invalid";
|
|
86
|
-
const cursor = page.nextCursor === undefined ? null : page.nextCursor;
|
|
87
|
-
if (!isJsonData(cursor) || (overlayCanonicalJson(cursor) ?? "").length > MAX_CURSOR_CHARS)
|
|
88
|
-
return "cursor_invalid";
|
|
89
|
-
return undefined;
|
|
90
|
-
}
|
|
91
|
-
/**
|
|
92
|
-
* Diff one page against the stored records. Within a page the last occurrence of an id wins and is
|
|
93
|
-
* diffed against the stored entry, so every change's `before` is what the overlay last accepted.
|
|
94
|
-
*/
|
|
95
|
-
function diffPage(state, page, bound, observedAt) {
|
|
96
|
-
const diagnostics = [];
|
|
97
|
-
const profileDiagnostics = [...(page.diagnostics ?? [])].map((d) => ({ ...d }));
|
|
98
|
-
const last = new Map();
|
|
99
|
-
const seen = new Set();
|
|
100
|
-
let notObjects = 0;
|
|
101
|
-
let duplicates = 0;
|
|
102
|
-
for (const raw of page.records) {
|
|
103
|
-
if (!record(raw)) {
|
|
104
|
-
notObjects += 1;
|
|
105
|
-
continue;
|
|
106
|
-
}
|
|
107
|
-
const mapped = mapRecord(bound.source, raw, bound.kind);
|
|
108
|
-
diagnostics.push(...mapped.diagnostics);
|
|
109
|
-
const id = mapped.external_id;
|
|
110
|
-
if (id === null)
|
|
111
|
-
continue;
|
|
112
|
-
seen.add(id);
|
|
113
|
-
if (last.has(id))
|
|
114
|
-
duplicates += 1;
|
|
115
|
-
last.set(id, mapped);
|
|
116
|
-
}
|
|
117
|
-
if (notObjects)
|
|
118
|
-
profileDiagnostics.push({ code: "RECORD_NOT_OBJECT", message: "records that are not JSON objects were skipped", count: notObjects });
|
|
119
|
-
if (duplicates)
|
|
120
|
-
profileDiagnostics.push({ code: "RECORD_DUPLICATE", message: "a page listed the same record more than once; the last one was used", count: duplicates });
|
|
121
|
-
const changes = [];
|
|
122
|
-
const entries = new Map();
|
|
123
|
-
for (const [id, mapped] of last) {
|
|
124
|
-
const stored = Object.hasOwn(state.records, id) ? state.records[id] : undefined;
|
|
125
|
-
if (mapped.kind === "refused") {
|
|
126
|
-
// Refused but present: still observed, so never missing; its accepted values stay.
|
|
127
|
-
if (stored)
|
|
128
|
-
entries.set(id, { ...stored, missing: 0, seen_at: observedAt, first_missing_at: undefined });
|
|
129
|
-
continue;
|
|
130
|
-
}
|
|
131
|
-
const previous = stored && !stored.gone ? stored.after : null;
|
|
132
|
-
const change = changeBetween(previous, mapped, { external_id: id, change_kind: previous === null ? "new" : "changed" });
|
|
133
|
-
entries.set(id, { after: change.after, digest: change.content_digest, missing: 0, seen_at: observedAt });
|
|
134
|
-
if (previous === null || change.changed_fields.length > 0)
|
|
135
|
-
changes.push(change);
|
|
136
|
-
}
|
|
137
|
-
return { changes, entries, seen, diagnostics, profileDiagnostics };
|
|
138
|
-
}
|
|
139
|
-
/** Disappearances at the end of a complete observation: every present record the pass did not list. */
|
|
140
|
-
function missingAfterComplete(records, seen, threshold, observedAt) {
|
|
141
|
-
const changes = [];
|
|
142
|
-
const entries = new Map();
|
|
143
|
-
for (const [id, entry] of Object.entries(records)) {
|
|
144
|
-
if (entry.gone || seen.has(id))
|
|
145
|
-
continue;
|
|
146
|
-
const missing = entry.missing + 1;
|
|
147
|
-
const firstMissingAt = entry.first_missing_at ?? observedAt;
|
|
148
|
-
if (missing >= threshold) {
|
|
149
|
-
entries.set(id, { ...entry, missing, first_missing_at: firstMissingAt, gone: true });
|
|
150
|
-
changes.push({
|
|
151
|
-
external_id: id, change_kind: "disappeared", changed_fields: [], before: { ...entry.after }, after: null,
|
|
152
|
-
content_digest: entry.digest, missing_count: missing, first_missing_at: firstMissingAt,
|
|
153
|
-
});
|
|
154
|
-
}
|
|
155
|
-
else {
|
|
156
|
-
entries.set(id, { ...entry, missing, first_missing_at: firstMissingAt });
|
|
157
|
-
}
|
|
158
|
-
}
|
|
159
|
-
return { changes, entries };
|
|
160
|
-
}
|
|
161
|
-
/** Drop `undefined` members so the stored state is plain JSON. */
|
|
162
|
-
function entryJson(entry) {
|
|
163
|
-
const { first_missing_at: firstMissingAt, gone, ...rest } = entry;
|
|
164
|
-
return { ...rest, ...(firstMissingAt !== undefined ? { first_missing_at: firstMissingAt } : {}), ...(gone ? { gone } : {}) };
|
|
165
|
-
}
|
|
166
49
|
function emptyState(profile) {
|
|
167
50
|
return { v: 1, profile: { id: profile.id, version: profile.version }, cursor: null, open: false, records: {}, complete_observations: 0 };
|
|
168
51
|
}
|
|
@@ -202,13 +85,6 @@ function findingUrl(url) {
|
|
|
202
85
|
return undefined;
|
|
203
86
|
}
|
|
204
87
|
}
|
|
205
|
-
/** The ids a commit did not write, read defensively. */
|
|
206
|
-
function commitResult(result) {
|
|
207
|
-
const ids = (value) => (Array.isArray(value) ? value.filter((id) => typeof id === "string") : []);
|
|
208
|
-
const retry = new Set(record(result) ? ids(result.retry) : []);
|
|
209
|
-
const hold = new Set((record(result) ? ids(result.hold) : []).filter((id) => !retry.has(id)));
|
|
210
|
-
return { hold, retry };
|
|
211
|
-
}
|
|
212
88
|
/**
|
|
213
89
|
* Run one observation of a bound Source Feed. Returns a receipt; never throws for a provider,
|
|
214
90
|
* profile, guard or commit problem, which all end the run with an outcome and a finding.
|
|
@@ -346,9 +222,6 @@ export async function observeSource(options) {
|
|
|
346
222
|
// A complete observation always lists from the start; an incremental one continues the stored cursor.
|
|
347
223
|
const resumed = mode === "incremental" && state.open;
|
|
348
224
|
receipt.pass.resumed = resumed;
|
|
349
|
-
let cursor = mode === "complete" ? null : state.cursor;
|
|
350
|
-
const threshold = bound.policy.missing_threshold ?? DEFAULT_MISSING_THRESHOLD;
|
|
351
|
-
const sourceComplete = bound.source.completeness === "complete";
|
|
352
225
|
// Without the caller's deadline the run has its own; every request has its own timeout either way.
|
|
353
226
|
const ownDeadline = options.signal ? undefined : new AbortController();
|
|
354
227
|
const timer = ownDeadline ? setTimeout(() => ownDeadline.abort(), budget.deadlineMs) : undefined;
|
|
@@ -366,249 +239,16 @@ export async function observeSource(options) {
|
|
|
366
239
|
signal, requestTimeoutMs: budget.requestTimeoutMs, authority: declarationRules, graphqlPaths: providerRequestRules(connector.auth)?.read_posts.filter(p => p.graphql).map(p => p.path),
|
|
367
240
|
});
|
|
368
241
|
receipt.requests = guard.requests;
|
|
369
|
-
/** Store `next` after a commit; false when the store refused it and the run has stopped. */
|
|
370
|
-
const store = async (next) => {
|
|
371
|
-
try {
|
|
372
|
-
version = await options.state.write(key, next, version);
|
|
373
|
-
state = next;
|
|
374
|
-
receipt.pending = Object.keys(next.pending ?? {}).length;
|
|
375
|
-
return true;
|
|
376
|
-
}
|
|
377
|
-
catch (error) {
|
|
378
|
-
const conflict = record(error) && error.code === "MACHINE_STATE_CONFLICT";
|
|
379
|
-
stop = { outcome: "failed", reason: conflict ? "state_conflict" : "state_unwritable", retry: true };
|
|
380
|
-
return false;
|
|
381
|
-
}
|
|
382
|
-
};
|
|
383
242
|
try {
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
for (const change of changes)
|
|
394
|
-
records[change.external_id] = { fields: { ...(change.after ?? change.before ?? {}) } };
|
|
395
|
-
let result;
|
|
396
|
-
let failure;
|
|
397
|
-
if (live.length) {
|
|
398
|
-
try {
|
|
399
|
-
result = commitResult(await options.commit({
|
|
400
|
-
feed: key, provider: bound.policy.provider, observed_at: observedAt, page: 0, replay: true, changes, records, baseline: false, complete: false,
|
|
401
|
-
}));
|
|
402
|
-
}
|
|
403
|
-
catch (error) {
|
|
404
|
-
failure = errorText(error);
|
|
405
|
-
}
|
|
406
|
-
}
|
|
407
|
-
const nextRecords = { ...state.records };
|
|
408
|
-
const nextPending = {};
|
|
409
|
-
const dropped = [];
|
|
410
|
-
for (const id of live) {
|
|
411
|
-
const entry = pending[id];
|
|
412
|
-
if (failure !== undefined) {
|
|
413
|
-
// A replay the commit cannot take stays pending, so a later sighting can still supersede
|
|
414
|
-
// it, until it has failed too often.
|
|
415
|
-
const failures = (entry.replay_failures ?? 0) + 1;
|
|
416
|
-
if (failures >= MAX_REPLAY_FAILURES)
|
|
417
|
-
dropped.push(id);
|
|
418
|
-
else
|
|
419
|
-
nextPending[id] = { ...entry, replay_failures: failures };
|
|
420
|
-
}
|
|
421
|
-
else if (result.retry.has(id)) {
|
|
422
|
-
nextPending[id] = entry;
|
|
423
|
-
}
|
|
424
|
-
else if (!result.hold.has(id)) {
|
|
425
|
-
// The record's absence since the hold is newer than the change, so its count stays; a
|
|
426
|
-
// record that comes back starts afresh.
|
|
427
|
-
const current = state.records[id];
|
|
428
|
-
nextRecords[id] = entryJson(current?.gone ? entry.entry : { ...entry.entry, missing: current?.missing ?? 0, first_missing_at: current?.first_missing_at });
|
|
429
|
-
}
|
|
430
|
-
}
|
|
431
|
-
receipt.held += live.filter((id) => failure !== undefined || result.retry.has(id) || result.hold.has(id)).length;
|
|
432
|
-
if (failure !== undefined) {
|
|
433
|
-
extraFindings.push({
|
|
434
|
-
key: `source:${bound.source.name}:replay_failed`,
|
|
435
|
-
title: clip(`Source ${bound.source.name}: a held change could not be planned again`, 140),
|
|
436
|
-
summary: clip(`Replaying ${live.length} pending change${live.length === 1 ? "" : "s"} failed (${failure}); they stay pending and the run went on to read the provider.`, 600),
|
|
437
|
-
priority: "normal",
|
|
438
|
-
facts: { outcome: "failed", reason: "replay_failed" },
|
|
439
|
-
});
|
|
440
|
-
}
|
|
441
|
-
if (dropped.length) {
|
|
442
|
-
extraFindings.push({
|
|
443
|
-
key: `source:${bound.source.name}:replay_dropped`,
|
|
444
|
-
title: clip(`Source ${bound.source.name}: held changes dropped after ${MAX_REPLAY_FAILURES} failed replays`, 140),
|
|
445
|
-
summary: clip(`Dropped ${dropped.length} pending change${dropped.length === 1 ? "" : "s"} (${dropped.slice(0, 3).map((id) => clip(id, 60)).join(", ")}${dropped.length > 3 ? ", …" : ""}); each record is planned again when it is next observed.`, 600),
|
|
446
|
-
priority: "normal",
|
|
447
|
-
facts: { outcome: "failed", reason: "replay_dropped", count: dropped.length },
|
|
448
|
-
});
|
|
449
|
-
}
|
|
450
|
-
const next = { ...state, records: nextRecords };
|
|
451
|
-
if (Object.keys(nextPending).length)
|
|
452
|
-
next.pending = nextPending;
|
|
453
|
-
else
|
|
454
|
-
delete next.pending;
|
|
455
|
-
if (!(await store(next)))
|
|
456
|
-
return finish();
|
|
457
|
-
}
|
|
458
|
-
const seen = new Set();
|
|
459
|
-
/** The cursors this run has asked for; a page handing one back again has looped. */
|
|
460
|
-
const asked = new Set();
|
|
461
|
-
const lastRequestUrl = () => guard.requests.at(-1)?.url;
|
|
462
|
-
for (let first = true;; first = false) {
|
|
463
|
-
if (signal.aborted) {
|
|
464
|
-
stop = { outcome: "deferred", reason: "deadline", retry: true };
|
|
465
|
-
break;
|
|
466
|
-
}
|
|
467
|
-
// A resumed pass may restart from its first page, so the cursor it resumes from is not a loop.
|
|
468
|
-
if (cursor !== null && !(first && resumed))
|
|
469
|
-
asked.add(overlayCanonicalJson(cursor));
|
|
470
|
-
let raw;
|
|
471
|
-
try {
|
|
472
|
-
// The deadline bounds profile code too: a page still pending when it passes is discarded.
|
|
473
|
-
raw = await raceAbort(profile.observe({
|
|
474
|
-
spec, bound: structuredClone(bound), cursor, resumed: first && resumed, mode, transport: guard.transport,
|
|
475
|
-
budget: { requests: guard.remaining(), maxBytes: budget.maxBytes }, now: now(),
|
|
476
|
-
validators: mode === "complete" ? null : state.validators ?? null,
|
|
477
|
-
}), signal);
|
|
478
|
-
}
|
|
479
|
-
catch (error) {
|
|
480
|
-
stop = guard.stopped() ?? (error instanceof SourceStop ? error.stop : undefined);
|
|
481
|
-
if (!stop && signal.aborted)
|
|
482
|
-
stop = { outcome: "deferred", reason: "deadline", retry: true };
|
|
483
|
-
if (!stop) {
|
|
484
|
-
const code = record(error) ? error.code : undefined;
|
|
485
|
-
stop = code === UNSUPPORTED_CONTENT
|
|
486
|
-
? { outcome: "unsupported_content", retry: false, detail: String(error.detail ?? "") }
|
|
487
|
-
: { outcome: "failed", reason: "profile_error", retry: false, detail: errorText(error) };
|
|
488
|
-
}
|
|
489
|
-
stopUrl = lastRequestUrl();
|
|
490
|
-
break;
|
|
491
|
-
}
|
|
492
|
-
// A stop the profile caught and swallowed still ends the run, and its page is discarded.
|
|
493
|
-
const guardStop = guard.stopped();
|
|
494
|
-
if (guardStop) {
|
|
495
|
-
stop = guardStop;
|
|
496
|
-
stopUrl = lastRequestUrl();
|
|
497
|
-
break;
|
|
498
|
-
}
|
|
499
|
-
let page;
|
|
500
|
-
try {
|
|
501
|
-
const invalid = pageProblem(raw);
|
|
502
|
-
if (invalid) {
|
|
503
|
-
stop = { outcome: "failed", reason: invalid, retry: false };
|
|
504
|
-
break;
|
|
505
|
-
}
|
|
506
|
-
// One read of the profile's page, as JSON data, so no getter or later mutation reaches the diff.
|
|
507
|
-
page = JSON.parse(JSON.stringify(raw));
|
|
508
|
-
}
|
|
509
|
-
catch (error) {
|
|
510
|
-
stop = { outcome: "failed", reason: "page_invalid", retry: false, detail: errorText(error) };
|
|
511
|
-
break;
|
|
512
|
-
}
|
|
513
|
-
const nextCursor = page.nextCursor ?? null;
|
|
514
|
-
// A page that promises more but hands back no cursor, or one this run already asked for, would
|
|
515
|
-
// read the same pages until the budget runs out.
|
|
516
|
-
if (page.more && (nextCursor === null || asked.has(overlayCanonicalJson(nextCursor)))) {
|
|
517
|
-
stop = { outcome: "failed", reason: "cursor_stuck", retry: false };
|
|
518
|
-
break;
|
|
519
|
-
}
|
|
520
|
-
const diff = diffPage(state, page, bound, observedAt);
|
|
521
|
-
for (const id of diff.seen)
|
|
522
|
-
seen.add(id);
|
|
523
|
-
const ended = !page.more;
|
|
524
|
-
const complete = ended && mode === "complete" && page.complete && sourceComplete;
|
|
525
|
-
const merged = { ...state.records };
|
|
526
|
-
for (const [id, entry] of diff.entries)
|
|
527
|
-
merged[id] = entry;
|
|
528
|
-
const missing = complete ? missingAfterComplete(merged, seen, threshold, observedAt) : { changes: [], entries: new Map() };
|
|
529
|
-
const changes = [...diff.changes, ...missing.changes];
|
|
530
|
-
const byId = new Map(changes.map((change) => [change.external_id, change]));
|
|
531
|
-
// A newer observation of a record, its listing or its absence from a complete pass, supersedes
|
|
532
|
-
// its pending change, whether or not the record was ever stored. Those entries are dropped and
|
|
533
|
-
// stored before the commit, so a crash after the commit never replays a change older than what
|
|
534
|
-
// the commit wrote.
|
|
535
|
-
const unlisted = complete ? Object.keys(state.pending ?? {}).filter((id) => !seen.has(id)) : [];
|
|
536
|
-
const superseded = [...new Set([...diff.seen, ...missing.entries.keys(), ...unlisted])].filter((id) => state.pending !== undefined && Object.hasOwn(state.pending, id));
|
|
537
|
-
if (superseded.length) {
|
|
538
|
-
const remaining = { ...state.pending };
|
|
539
|
-
for (const id of superseded)
|
|
540
|
-
delete remaining[id];
|
|
541
|
-
const pre = { ...state };
|
|
542
|
-
if (Object.keys(remaining).length)
|
|
543
|
-
pre.pending = remaining;
|
|
544
|
-
else
|
|
545
|
-
delete pre.pending;
|
|
546
|
-
if (!(await store(pre)))
|
|
547
|
-
break;
|
|
548
|
-
}
|
|
549
|
-
const records = {};
|
|
550
|
-
for (const change of changes)
|
|
551
|
-
records[change.external_id] = { fields: { ...(change.after ?? change.before ?? {}) } };
|
|
552
|
-
let result;
|
|
553
|
-
try {
|
|
554
|
-
result = commitResult(await options.commit({
|
|
555
|
-
feed: key, provider: bound.policy.provider, observed_at: observedAt, page: receipt.pages + 1, replay: false,
|
|
556
|
-
changes, records, baseline, complete,
|
|
557
|
-
}));
|
|
558
|
-
}
|
|
559
|
-
catch (error) {
|
|
560
|
-
stop = { outcome: "failed", reason: "commit_failed", retry: true, detail: errorText(error) };
|
|
561
|
-
break;
|
|
562
|
-
}
|
|
563
|
-
const notWritten = (id) => byId.has(id) && (result.hold.has(id) || result.retry.has(id));
|
|
564
|
-
// The page committed: now, and only now, its records and cursor are stored.
|
|
565
|
-
const nextRecords = { ...state.records };
|
|
566
|
-
const nextPending = { ...state.pending };
|
|
567
|
-
for (const [id, entry] of [...diff.entries, ...missing.entries]) {
|
|
568
|
-
if (byId.has(id) && result.retry.has(id))
|
|
569
|
-
nextPending[id] = { change: byId.get(id), entry: entryJson(entry), held_at: observedAt };
|
|
570
|
-
else if (!notWritten(id))
|
|
571
|
-
nextRecords[id] = entryJson(entry);
|
|
572
|
-
}
|
|
573
|
-
const next = {
|
|
574
|
-
...state,
|
|
575
|
-
cursor: nextCursor,
|
|
576
|
-
open: page.more,
|
|
577
|
-
records: nextRecords,
|
|
578
|
-
complete_observations: state.complete_observations + (complete ? 1 : 0),
|
|
579
|
-
...(complete ? { last_complete_at: observedAt } : {}),
|
|
580
|
-
...(page.validators ? { validators: { ...page.validators } } : {}),
|
|
581
|
-
};
|
|
582
|
-
if (Object.keys(nextPending).length)
|
|
583
|
-
next.pending = nextPending;
|
|
584
|
-
else
|
|
585
|
-
delete next.pending;
|
|
586
|
-
if (!(await store(next)))
|
|
587
|
-
break;
|
|
588
|
-
receipt.pages += 1;
|
|
589
|
-
receipt.cursor = nextCursor;
|
|
590
|
-
receipt.diagnostics.push(...diff.diagnostics);
|
|
591
|
-
receipt.profile_diagnostics.push(...diff.profileDiagnostics);
|
|
592
|
-
for (const change of changes) {
|
|
593
|
-
if (notWritten(change.external_id)) {
|
|
594
|
-
receipt.held += 1;
|
|
595
|
-
continue;
|
|
596
|
-
}
|
|
597
|
-
const kind = change.change_kind;
|
|
598
|
-
if (Object.hasOwn(receipt.changes, kind))
|
|
599
|
-
receipt.changes[kind] += 1;
|
|
600
|
-
}
|
|
601
|
-
if (ended) {
|
|
602
|
-
receipt.pass.ended = true;
|
|
603
|
-
receipt.pass.complete = complete;
|
|
604
|
-
break;
|
|
605
|
-
}
|
|
606
|
-
if (receipt.pages >= budget.pages) {
|
|
607
|
-
stop = { outcome: "deferred", reason: "budget", retry: true };
|
|
608
|
-
break;
|
|
609
|
-
}
|
|
610
|
-
cursor = nextCursor;
|
|
611
|
-
}
|
|
243
|
+
await driveObservation({
|
|
244
|
+
bound, state: options.state, initial: state, version, receipt, observedAt, mode, resumed, signal, pages: budget.pages,
|
|
245
|
+
acquire: ({ cursor, resumed, mode, validators }) => raceAbort(profile.observe({
|
|
246
|
+
spec, bound: structuredClone(bound), cursor, resumed, mode, transport: guard.transport,
|
|
247
|
+
budget: { requests: guard.remaining(), maxBytes: budget.maxBytes }, now: now(), validators,
|
|
248
|
+
}), signal),
|
|
249
|
+
stopped: () => guard.stopped(), lastUrl: () => guard.requests.at(-1)?.url,
|
|
250
|
+
stop: (value, url) => { stop = value; stopUrl = url; }, extraFindings, commit: options.commit,
|
|
251
|
+
});
|
|
612
252
|
return finish();
|
|
613
253
|
}
|
|
614
254
|
finally {
|