@superbee/core 0.2.0-pre.3 → 0.2.0-pre.31

package/dist/source-engine/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/source-engine/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AACH,OAAO,EACL,mBAAmB,EACnB,aAAa,EACb,WAAW,EACX,cAAc,EACd,cAAc,EACd,iBAAiB,GAClB,MAAM,aAAa,CAAC;AACrB,YAAY,EAAE,aAAa,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAC9D,OAAO,EAAE,qBAAqB,EAAE,oBAAoB,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AACxF,YAAY,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAC1C,OAAO,EAAE,yBAAyB,EAAE,gBAAgB,EAAE,mBAAmB,EAAE,aAAa,EAAE,kBAAkB,EAAE,MAAM,cAAc,CAAC;AACnI,OAAO,EAAE,aAAa,EAAE,oBAAoB,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AAC7F,YAAY,EAAE,YAAY,EAAE,aAAa,EAAE,WAAW,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAChG,YAAY,EACV,YAAY,EACZ,UAAU,EACV,gBAAgB,EAChB,cAAc,EACd,WAAW,EACX,oBAAoB,EACpB,UAAU,EACV,iBAAiB,EACjB,cAAc,EACd,eAAe,EACf,gBAAgB,EAChB,YAAY,EACZ,YAAY,EACZ,aAAa,EACb,aAAa,EACb,gBAAgB,EAChB,gBAAgB,EAChB,UAAU,EACV,WAAW,EACX,SAAS,EACT,gBAAgB,EAChB,eAAe,GAChB,MAAM,YAAY,CAAC;AACpB,OAAO,EAAE,0BAA0B,EAAE,uBAAuB,EAAE,0BAA0B,EAAE,MAAM,cAAc,CAAC;AAC/G,YAAY,EAAE,oBAAoB,EAAE,kBAAkB,EAAE,oBAAoB,EAAE,mBAAmB,EAAE,qBAAqB,EAAE,MAAM,cAAc,CAAC"}

package/dist/source-engine/index.js

@@ -0,0 +1,21 @@
+/**
+ * `@superbee/core/source-engine`: the portable half of the integration source engine.
+ *
+ * A {@link SourceProfile} turns a bound Source Feed, a cursor and a transport into pages of
+ * normalized records; it never opens a socket, never sees a credential and never writes. A
+ * {@link Transport} is the one network port: it never follows a redirect and refuses a body over
+ * its byte bound. {@link observeSource} wraps the transport before a profile sees it and owns the
+ * host guard (every URL and redirect hop within the profile's hosts, the auth kind's credential
+ * hosts and the declared keyless domains, plus the address rule), robots.txt for keyless hosts,
+ * the method rule and header allowlist, the budget and deadline, the status-to-outcome mapping,
+ * the cursor rule (a cursor is stored only after its page commits), completeness, and the diff into
+ * record-overlay changes, with disappearance only from a complete observation.
+ *
+ * Storage-neutral and Worker-free: no Node globals or builtins, and nothing here calls `fetch`.
+ */
+export { canonicalRobotsPath, fetchIdentity, parseRobots, refusedAddress, robotsDecision, robotsRuleMatches, } from "./robots.js";
+export { DEFAULT_SOURCE_BUDGET, DEFAULT_STATUS_RULES, sourceHostSet } from "./guard.js";
+export { DEFAULT_MISSING_THRESHOLD, MAX_CURSOR_CHARS, MAX_REPLAY_FAILURES, observeSource, unsupportedContent } from "./observe.js";
+export { isSourceState, MachineStateConflict, MemoryMachineState } from "./machine-state.js";
+export { createSourceOverlayAdapter, isSourceOverlaySnapshot, MAX_SOURCE_OVERLAY_ENTRIES } from "./overlay.js";
+//# sourceMappingURL=index.js.map

package/dist/source-engine/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/source-engine/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AACH,OAAO,EACL,mBAAmB,EACnB,aAAa,EACb,WAAW,EACX,cAAc,EACd,cAAc,EACd,iBAAiB,GAClB,MAAM,aAAa,CAAC;AAErB,OAAO,EAAE,qBAAqB,EAAE,oBAAoB,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAExF,OAAO,EAAE,yBAAyB,EAAE,gBAAgB,EAAE,mBAAmB,EAAE,aAAa,EAAE,kBAAkB,EAAE,MAAM,cAAc,CAAC;AACnI,OAAO,EAAE,aAAa,EAAE,oBAAoB,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AA0B7F,OAAO,EAAE,0BAA0B,EAAE,uBAAuB,EAAE,0BAA0B,EAAE,MAAM,cAAc,CAAC"}

package/dist/source-engine/machine-state.d.ts

@@ -0,0 +1,74 @@
+/**
+ * The machine-state port: one JSON document per Source Feed holding the stored cursor, the last
+ * accepted values of every record and the changes still pending a write, written with a
+ * compare-and-swap on its version. The managed runner keeps it in its own store;
+ * `MemoryMachineState` serves tests and local runs.
+ */
+import type { RecordChange } from "../record-overlay/types.js";
+/** One record's last accepted state. */
+export interface RecordEntry {
+    /** The last accepted `after`, carried values included. */
+    after: Record<string, unknown>;
+    digest: string;
+    /** Consecutive complete observations without the record. */
+    missing: number;
+    first_missing_at?: string;
+    /** Reported `disappeared`; a later sighting is `new` again. */
+    gone?: true;
+    seen_at: string;
+}
+/** A change a commit asked to retry: planned again at the start of every run until written. */
+export interface PendingChange {
+    change: RecordChange;
+    /** The record's entry once the change is written. */
+    entry: RecordEntry;
+    held_at: string;
+    /** Replays whose commit threw. */
+    replay_failures?: number;
+}
+export interface SourceState {
+    v: 1;
+    /** The profile that wrote `cursor`; a cursor from another id or version is never handed back. */
+    profile: {
+        id: string;
+        version: number;
+    };
+    cursor: unknown;
+    /** The last committed page left its pass open. */
+    open: boolean;
+    validators?: Record<string, string>;
+    records: Record<string, RecordEntry>;
+    pending?: Record<string, PendingChange>;
+    complete_observations: number;
+    last_complete_at?: string;
+}
+export interface MachineState {
+    /** The state stored under `key` and the version a write must name, or null. */
+    read(key: string): Promise<{
+        state: SourceState;
+        version: string;
+    } | null>;
+    /**
+     * Store `state` when the stored version is still `expected` (null: nothing stored) and return the
+     * new version; otherwise throw a {@link MachineStateConflict}.
+     */
+    write(key: string, state: SourceState, expected: string | null): Promise<string>;
+}
+export declare class MachineStateConflict extends Error {
+    readonly code = "MACHINE_STATE_CONFLICT";
+    constructor(key: string);
+}
+/** Machine state in memory. Values are copied in and out, so a caller never aliases the store. */
+export declare class MemoryMachineState implements MachineState {
+    private readonly entries;
+    private counter;
+    read(key: string): Promise<{
+        state: SourceState;
+        version: string;
+    } | null>;
+    write(key: string, state: SourceState, expected: string | null): Promise<string>;
+    /** Every key stored. */
+    keys(): string[];
+}
+export declare function isSourceState(value: unknown): value is SourceState;
+//# sourceMappingURL=machine-state.d.ts.map

package/dist/source-engine/machine-state.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"machine-state.d.ts","sourceRoot":"","sources":["../../src/source-engine/machine-state.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AACH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAC;AAE/D,wCAAwC;AACxC,MAAM,WAAW,WAAW;IAC1B,0DAA0D;IAC1D,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC/B,MAAM,EAAE,MAAM,CAAC;IACf,4DAA4D;IAC5D,OAAO,EAAE,MAAM,CAAC;IAChB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,+DAA+D;IAC/D,IAAI,CAAC,EAAE,IAAI,CAAC;IACZ,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,+FAA+F;AAC/F,MAAM,WAAW,aAAa;IAC5B,MAAM,EAAE,YAAY,CAAC;IACrB,qDAAqD;IACrD,KAAK,EAAE,WAAW,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC;IAChB,kCAAkC;IAClC,eAAe,CAAC,EAAE,MAAM,CAAC;CAC1B;AAED,MAAM,WAAW,WAAW;IAC1B,CAAC,EAAE,CAAC,CAAC;IACL,iGAAiG;IACjG,OAAO,EAAE;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE,CAAC;IACzC,MAAM,EAAE,OAAO,CAAC;IAChB,kDAAkD;IAClD,IAAI,EAAE,OAAO,CAAC;IACd,UAAU,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACpC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;IACrC,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC;IACxC,qBAAqB,EAAE,MAAM,CAAC;IAC9B,gBAAgB,CAAC,EAAE,MAAM,CAAC;CAC3B;AAED,MAAM,WAAW,YAAY;IAC3B,+EAA+E;IAC/E,IAAI,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;QAAE,KAAK,EAAE,WAAW,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI,CAAC,CAAC;IAC3E;;;OAGG;IACH,KAAK,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,MAAM,GAAG,IAAI,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;CAClF;AAED,qBAAa,oBAAqB,SAAQ,KAAK;IAC7C,QAAQ,CAAC,IAAI,4BAA4B;gBAC7B,GAAG,EAAE,MAAM;CAIxB;AAED,kGAAkG;AAClG,qBAAa,kBAAmB,YAAW,YAAY;IACrD,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAwD;IAChF,OAAO,CAAC,OAAO,CAAK;IAEd,IAAI,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;QAAE,KAAK,EAAE,WAAW,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI,CAAC;IAK1E,KAAK,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,MAAM,GAAG,IAAI,GAAG,OAAO,CAAC,MAAM,CAAC;IAQtF,wBAAwB;IACxB,IAAI,IAAI,MAAM,EAAE;CAGjB;AASD,wBAAgB,aAAa,CAAC,KAAK,EAAE,OAAO,GAAG,KAAK,IAAI,WAAW,CASlE"}

package/dist/source-engine/machine-state.js

@@ -0,0 +1,48 @@
+export class MachineStateConflict extends Error {
+    code = "MACHINE_STATE_CONFLICT";
+    constructor(key) {
+        super(`machine state for ${key} changed since it was read`);
+        this.name = "MachineStateConflict";
+    }
+}
+/** Machine state in memory. Values are copied in and out, so a caller never aliases the store. */
+export class MemoryMachineState {
+    entries = new Map();
+    counter = 0;
+    async read(key) {
+        const entry = this.entries.get(key);
+        return entry ? { state: JSON.parse(entry.json), version: entry.version } : null;
+    }
+    async write(key, state, expected) {
+        const current = this.entries.get(key)?.version ?? null;
+        if (current !== expected)
+            throw new MachineStateConflict(key);
+        const version = String(++this.counter);
+        this.entries.set(key, { json: JSON.stringify(state), version });
+        return version;
+    }
+    /** Every key stored. */
+    keys() {
+        return [...this.entries.keys()].sort();
+    }
+}
+const record = (value) => value !== null && typeof value === "object" && !Array.isArray(value);
+/** Whether a stored value has the shape the engine writes. */
+const isEntry = (entry) => record(entry) && record(entry.after) && typeof entry.digest === "string"
+    && Number.isInteger(entry.missing) && typeof entry.seen_at === "string";
+export function isSourceState(value) {
+    if (!record(value) || value.v !== 1 || typeof value.open !== "boolean" || !record(value.records))
+        return false;
+    if (!record(value.profile) || typeof value.profile.id !== "string" || !Number.isInteger(value.profile.version))
+        return false;
+    if (!Number.isInteger(value.complete_observations))
+        return false;
+    if (value.validators !== undefined && !record(value.validators))
+        return false;
+    if (value.pending !== undefined && !(record(value.pending) && Object.entries(value.pending).every(([id, pending]) => record(pending)
+        && isEntry(pending.entry) && typeof pending.held_at === "string" && record(pending.change) && pending.change.external_id === id
+        && typeof pending.change.change_kind === "string")))
+        return false;
+    return Object.values(value.records).every(isEntry);
+}
+//# sourceMappingURL=machine-state.js.map

package/dist/source-engine/machine-state.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"machine-state.js","sourceRoot":"","sources":["../../src/source-engine/machine-state.ts"],"names":[],"mappings":"AAuDA,MAAM,OAAO,oBAAqB,SAAQ,KAAK;IACpC,IAAI,GAAG,wBAAwB,CAAC;IACzC,YAAY,GAAW;QACrB,KAAK,CAAC,qBAAqB,GAAG,4BAA4B,CAAC,CAAC;QAC5D,IAAI,CAAC,IAAI,GAAG,sBAAsB,CAAC;IACrC,CAAC;CACF;AAED,kGAAkG;AAClG,MAAM,OAAO,kBAAkB;IACZ,OAAO,GAAG,IAAI,GAAG,EAA6C,CAAC;IACxE,OAAO,GAAG,CAAC,CAAC;IAEpB,KAAK,CAAC,IAAI,CAAC,GAAW;QACpB,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACpC,OAAO,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,CAAgB,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;IACjG,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,GAAW,EAAE,KAAkB,EAAE,QAAuB;QAClE,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,OAAO,IAAI,IAAI,CAAC;QACvD,IAAI,OAAO,KAAK,QAAQ;YAAE,MAAM,IAAI,oBAAoB,CAAC,GAAG,CAAC,CAAC;QAC9D,MAAM,OAAO,GAAG,MAAM,CAAC,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;QACvC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC;QAChE,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,wBAAwB;IACxB,IAAI;QACF,OAAO,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;IACzC,CAAC;CACF;AAED,MAAM,MAAM,GAAG,CAAC,KAAc,EAAoC,EAAE,CAClE,KAAK,KAAK,IAAI,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;AAEvE,8DAA8D;AAC9D,MAAM,OAAO,GAAG,CAAC,KAAc,EAAW,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,OAAO,KAAK,CAAC,MAAM,KAAK,QAAQ;OAChH,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,OAAO,KAAK,CAAC,OAAO,KAAK,QAAQ,CAAC;AAE1E,MAAM,UAAU,aAAa,CAAC,KAAc;IAC1C,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,CAAC,KAAK,CAAC,IAAI,OAAO,KAAK,CAAC,IAAI,KAAK,SAAS,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC;QAAE,OAAO,KAAK,CAAC;IAC/G,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,OAAO,KAAK,CAAC,OAAO,CAAC,EAAE,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC;QAAE,OAAO,KAAK,CAAC;IAC7H,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,qBAAqB,CAAC;QAAE,OAAO,KAAK,CAAC;IACjE,IAAI,KAAK,CAAC,UAAU,KAAK,SAAS,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC;QAAE,OAAO,KAAK,CAAC;IAC9E,IAAI,KAAK,CAAC,OAAO,KAAK,SAAS,IAAI,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,OAAO,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC;WAC/H,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,OAAO,OAAO,CAAC,OAAO,KAAK,QAAQ,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,OAAO,CAAC,MAAM,CAAC,WAAW,KAAK,EAAE;WAC5H,OAAO,OAAO,CAAC,MAAM,CAAC,WAAW,KAAK,QAAQ,CAAC,CAAC;QAAE,OAAO,KAAK,CAAC;IACpE,OAAO,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;AACrD,CAAC"}

package/dist/source-engine/observe.d.ts

@@ -0,0 +1,18 @@
+import type { ObserveSourceOptions, SourceRunReceipt } from "./types.js";
+/** Complete observations without a record before it is `disappeared`, when the feed names none. */
+export declare const DEFAULT_MISSING_THRESHOLD = 2;
+/** The largest stored cursor, as canonical JSON characters. */
+export declare const MAX_CURSOR_CHARS = 8192;
+/**
+ * Replays of a pending change whose commit threw, after which the change is dropped with a finding:
+ * the record is planned again when it is next observed.
+ */
+export declare const MAX_REPLAY_FAILURES = 5;
+/** Thrown by a profile that cannot read a provider's body; the run ends `unsupported_content`. */
+export declare function unsupportedContent(detail: string): never;
+/**
+ * Run one observation of a bound Source Feed. Returns a receipt; never throws for a provider,
+ * profile, guard or commit problem, which all end the run with an outcome and a finding.
+ */
+export declare function observeSource<Spec, Cursor>(options: ObserveSourceOptions<Spec, Cursor>): Promise<SourceRunReceipt>;
+//# sourceMappingURL=observe.d.ts.map

package/dist/source-engine/observe.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"observe.d.ts","sourceRoot":"","sources":["../../src/source-engine/observe.ts"],"names":[],"mappings":"AAgBA,OAAO,KAAK,EAEV,oBAAoB,EAIpB,gBAAgB,EAEjB,MAAM,YAAY,CAAC;AAEpB,mGAAmG;AACnG,eAAO,MAAM,yBAAyB,IAAI,CAAC;AAC3C,+DAA+D;AAC/D,eAAO,MAAM,gBAAgB,OAAO,CAAC;AACrC;;;GAGG;AACH,eAAO,MAAM,mBAAmB,IAAI,CAAC;AAKrC,kGAAkG;AAClG,wBAAgB,kBAAkB,CAAC,MAAM,EAAE,MAAM,GAAG,KAAK,CAExD;AAiLD;;;GAGG;AACH,wBAAsB,aAAa,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,oBAAoB,CAAC,IAAI,EAAE,MAAM,CAAC,GAAG,OAAO,CAAC,gBAAgB,CAAC,CA2VxH"}