@opendatalabs/personal-server-ts-core 1.32.0 → 1.33.0

package/dist/additions/first-added.d.ts

@@ -0,0 +1,203 @@
+/**
+ * First-seen ledger: when each record of a scope was first seen.
+ *
+ * The Personal Server stores each data import as one envelope per scope,
+ * `{ scope, collectedAt, data }`, and a re-import replaces the whole scope
+ * with a fresh full snapshot. Nothing in that shape records when an
+ * individual record first appeared, so this module derives a per-record
+ * first-seen ledger by folding the scope's stored versions.
+ *
+ * The ledger is a SIDECAR kept beside the data (see `ledger-store.ts`). It
+ * is never stamped into a stored envelope, never synced as data and never
+ * served to grantees: it is a cache that can always be rebuilt from the
+ * retained versions.
+ *
+ * Everything here is pure: it never reads or writes a store and never
+ * mutates the objects it is handed. Folding is order independent and
+ * idempotent, so two devices that hold the same versions compute the same
+ * ledger, and a version folded twice changes nothing.
+ */
+/**
+ * At most this many tracked keys per scope snapshot AND in the ledger (the
+ * union over time). A snapshot over it is untrackable; when the union passes
+ * it, keys absent from the newest tracked version are dropped, oldest first.
+ */
+export declare const MAX_TRACKED_KEYS = 200000;
+/** Ids longer than this are replaced by a hash so keys stay bounded. */
+export declare const MAX_ID_LENGTH = 64;
+/** Collection names longer than this are replaced by a hash (they can be caller-controlled). */
+export declare const MAX_COLLECTION_LENGTH = 64;
+/** A record absent from the newest tracked version is dropped after this long. */
+export declare const PRUNE_AFTER_MS: number;
+/** An unreadable version is read again at most this often. */
+export declare const RETRY_UNREADABLE_MS: number;
+/** At most this many unreadable versions are remembered for retry. */
+export declare const MAX_RETRY_VERSIONS = 20;
+/** Items processed between two yields to the event loop. */
+export declare const YIELD_EVERY = 2000;
+/** Run the rest of an async loop in a later macrotask so other requests are served. */
+export declare function yieldToEventLoop(): Promise<void>;
+/**
+ * Why a ledger holds no per-record keys: a rebuild could not read any version
+ * within its byte budget (`too_large`), or none could be read (`unreadable`).
+ * A negative result persisted so it is not retried on every request; a newer
+ * retained version retries it. (A snapshot over the key cap is not a state of
+ * the ledger: that one version is simply untrackable, see `isTrackable`.)
+ */
+export type LedgerSkipReason = "too_large" | "unreadable";
+/** One document per scope, stored as a sidecar beside the data. */
+export interface ScopeFirstSeenLedger {
+    version: 3;
+    scope: string;
+    /**
+     * collectedAt of the EARLIEST TRACKABLE version folded in (one with records
+     * that have ids, or an empty one): records first seen then predate tracking.
+     * Null until a trackable version exists, so a binary, over-cap or
+     * id-less first snapshot never makes everything after it look added.
+     */
+    baseline: string | null;
+    /** collectedAt of the NEWEST trackable version: a record is present iff last seen then. */
+    current: string | null;
+    /** collectedAt of the NEWEST version of any kind, and its record total (a binary file counts 1). */
+    latest: {
+        collectedAt: string;
+        total: number;
+    };
+    /**
+     * The ledger is known complete up to this collectedAt: every retained
+     * version at or before it has been folded. A write that folds only itself
+     * advances it only when no other version is newer than it, so the owner's
+     * `/additions` read knows what to catch up on.
+     */
+    through: string;
+    /**
+     * Versions that could not be read when they were folded, and when the last
+     * attempt was made. They are retried at most once per `RETRY_UNREADABLE_MS`
+     * and meanwhile the ledger is served as it is (the numbers of the newest
+     * readable version) without reading anything. `through` has already moved
+     * past them, so a poll with nothing new reads nothing.
+     */
+    retry?: {
+        versions: string[];
+        attemptedAt: string;
+    };
+    /** Set when no per-record keys are kept (see `LedgerSkipReason`); `records` is then empty. */
+    skipped?: LedgerSkipReason;
+    /**
+     * The ledger was rebuilt from only the newest retained versions (count or
+     * byte budget). A version older than `baseline` then never moves the
+     * baseline back over the versions that were not folded.
+     */
+    partial?: true;
+    /** record key -> [firstSeen collectedAt, lastSeen collectedAt] */
+    records: Record<string, [string, string]>;
+}
+/** A stored version to fold: its collectedAt and the envelope `data` as stored. */
+export interface VersionToFold {
+    scope: string;
+    collectedAt: string;
+    data: Record<string, unknown>;
+}
+export interface RecordKeyExtraction {
+    /** De-duplicated keys of the records that have a usable id, in first-seen order. Empty when `tooMany`. */
+    keys: string[];
+    /** Records in the counted collections, tracked or not. */
+    total: number;
+    /** More than `MAX_TRACKED_KEYS` tracked keys: extraction stopped collecting them. */
+    tooMany?: true;
+}
+/**
+ * Stable keys for the records in an envelope `data` object, plus the number of
+ * records counted. Returns null when the data is not trackable (binary).
+ *
+ * Both stored forms of a scope give the same keys: the PDPP rows form
+ * `{ records: [row, ...] }` and the legacy keyed form `{ <collection>: [...] }`
+ * (a ruled scope names the id field per form, see `record-rules.ts`). A record
+ * is tracked only when it has a usable id; records without one count toward
+ * `total` but produce no key. Ids and collection names over 64 characters are
+ * hashed, so every key is at most about 200 characters.
+ */
+export declare function extractRecordKeys(scope: string, data: Record<string, unknown>): Promise<RecordKeyExtraction | null>;
+/**
+ * Folds stored versions into one ledger, one at a time, so a caller can read a
+ * version, `add` it and drop it before reading the next: only one parsed
+ * envelope is ever alive. Order independent and idempotent for versions within
+ * the 90-day pruning window of each other (pruning is relative to the newest
+ * tracked version, so folding versions further apart in a different order can
+ * re-date a record that was already pruned).
+ */
+export declare class LedgerFold {
+    private scope;
+    private baseline;
+    private current;
+    private latest;
+    private through;
+    private retry;
+    private skipped;
+    private partial;
+    private records;
+    constructor(ledger: ScopeFirstSeenLedger | null, scope?: string);
+    add(version: VersionToFold): Promise<void>;
+    /**
+     * Insert the records this version brings that the ledger does not know.
+     * Capacity is decided ONCE: when they do not fit, absent records are evicted
+     * in a single pass (see `evict`), and whatever still does not fit is left
+     * out, smallest keys first, so the result never depends on the order of the
+     * ids in the version.
+     */
+    private insertFresh;
+    /**
+     * Make room for `need` records by dropping some that are absent: not in the
+     * newest tracked version, not in the version being folded, and not
+     * pre-tracking (first seen at or before the baseline). Pre-tracking records
+     * are never dropped. Dated records that are absent CAN be dropped (and are
+     * dated as new if they return): that needs a scope near the cap, and
+     * whoever can write the scope can provoke it. Among the evictable ones the
+     * most recently first seen go first. Returns how many were dropped.
+     */
+    private evict;
+    /** Drop records that are not present and were last seen over 90 days before `current`. */
+    private prune;
+    /**
+     * Prune and return the ledger, or null when nothing was ever folded.
+     * `partial` marks a ledger rebuilt from only the newest retained versions;
+     * `through` overrides the completeness marker (see `ScopeFirstSeenLedger`).
+     */
+    finish(options?: {
+        partial?: boolean;
+        through?: string;
+        /** `null` clears the retry list; undefined keeps it. */
+        retry?: ScopeFirstSeenLedger["retry"] | null;
+    }): Promise<ScopeFirstSeenLedger | null>;
+}
+/**
+ * Fold several stored versions into a ledger in one pass. Equivalent to
+ * calling `foldVersion` for each, but copies the ledger and prunes once.
+ * Returns null only when no version had a parseable `collectedAt` and there
+ * was no starting ledger.
+ */
+export declare function foldVersions(ledger: ScopeFirstSeenLedger | null, versions: readonly VersionToFold[]): Promise<ScopeFirstSeenLedger | null>;
+/**
+ * Fold one stored version into a ledger (or start one). A version whose
+ * `collectedAt` does not parse is ignored.
+ */
+export declare function foldVersion(ledger: ScopeFirstSeenLedger | null, version: VersionToFold): Promise<ScopeFirstSeenLedger | null>;
+/** True when the record was in the newest tracked version folded in. */
+export declare function isPresent(ledger: ScopeFirstSeenLedger, key: string): boolean;
+/** True when the record was first seen at or before the baseline (or there is no baseline). */
+export declare function isPreTracking(ledger: ScopeFirstSeenLedger, key: string): boolean;
+/**
+ * First-seen timestamps of the records that count as additions: present in
+ * the newest tracked version and first seen after the baseline. One per
+ * record. A skipped ledger, one with no baseline yet, and a scope whose newest
+ * version is not its newest tracked one (a binary file or an over-cap
+ * snapshot, whose `latest.total` does not describe those records) report none,
+ * so `added` can never exceed `total`.
+ */
+export declare function listAddedTimestamps(ledger: ScopeFirstSeenLedger): string[];
+/**
+ * The ledger in `value`, or null unless it is a well-formed version-2
+ * document (older versions are rejected and rebuilt). Never throws. The result is a fresh object.
+ */
+export declare function readScopeFirstSeenLedger(value: unknown): ScopeFirstSeenLedger | null;
+//# sourceMappingURL=first-added.d.ts.map

package/dist/additions/first-added.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"first-added.d.ts","sourceRoot":"","sources":["../../src/additions/first-added.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAIH;;;;GAIG;AACH,eAAO,MAAM,gBAAgB,SAAU,CAAC;AACxC,wEAAwE;AACxE,eAAO,MAAM,aAAa,KAAK,CAAC;AAChC,gGAAgG;AAChG,eAAO,MAAM,qBAAqB,KAAK,CAAC;AACxC,kFAAkF;AAClF,eAAO,MAAM,cAAc,QAA2B,CAAC;AACvD,8DAA8D;AAC9D,eAAO,MAAM,mBAAmB,QAAiB,CAAC;AAClD,sEAAsE;AACtE,eAAO,MAAM,kBAAkB,KAAK,CAAC;AACrC,4DAA4D;AAC5D,eAAO,MAAM,WAAW,OAAQ,CAAC;AAEjC,uFAAuF;AACvF,wBAAgB,gBAAgB,IAAI,OAAO,CAAC,IAAI,CAAC,CAEhD;AAED;;;;;;GAMG;AACH,MAAM,MAAM,gBAAgB,GAAG,WAAW,GAAG,YAAY,CAAC;AAE1D,mEAAmE;AACnE,MAAM,WAAW,oBAAoB;IACnC,OAAO,EAAE,CAAC,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd;;;;;OAKG;IACH,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,2FAA2F;IAC3F,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IACvB,oGAAoG;IACpG,MAAM,EAAE;QAAE,WAAW,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC;IAC/C;;;;;OAKG;IACH,OAAO,EAAE,MAAM,CAAC;IAChB;;;;;;OAMG;IACH,KAAK,CAAC,EAAE;QAAE,QAAQ,EAAE,MAAM,EAAE,CAAC;QAAC,WAAW,EAAE,MAAM,CAAA;KAAE,CAAC;IACpD,8FAA8F;IAC9F,OAAO,CAAC,EAAE,gBAAgB,CAAC;IAC3B;;;;OAIG;IACH,OAAO,CAAC,EAAE,IAAI,CAAC;IACf,kEAAkE;IAClE,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;CAC3C;AAED,mFAAmF;AACnF,MAAM,WAAW,aAAa;IAC5B,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,CAAC;IACpB,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CAC/B;AAED,MAAM,WAAW,mBAAmB;IAClC,0GAA0G;IAC1G,IAAI,EAAE,MAAM,EAAE,CAAC;IACf,0DAA0D;IAC1D,KAAK,EAAE,MAAM,CAAC;IACd,qFAAqF;IACrF,OAAO,CAAC,EAAE,IAAI,CAAC;CAChB;AA8KD;;;;;;;;;;GAUG;AACH,wBAAsB,iBAAiB,CACrC,KAAK,EAAE,MAAM,EACb,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAC5B,OAAO,CAAC,mBAAmB,GAAG,IAAI,CAAC,CAwErC;AA2CD;;;;;;;GAOG;AACH,qBAAa,UAAU;IACrB,OAAO,CAAC,KAAK,CAAgB;IAC7B,OAAO,CAAC,QAAQ,CAAuB;IACvC,OAAO,CAAC,OAAO,CAAuB;IACtC,OAAO,CAAC,MAAM,CAA+C;IAC7D,OAAO,CAAC,OAAO,CAAuB;IACtC,OAAO,CAAC,KAAK,CAAgC;IAC7C,OAAO,CAAC,OAAO,CAA+B;IAC9C,OAAO,CAAC,OAAO,CAAS;IAGxB,OAAO,CAAC,OAAO,CAA2B;gBAE9B,MAAM,EAAE,oBAAoB,GAAG,IAAI,EAAE,KAAK,CAAC,EAAE,MAAM;IAqBzD,GAAG,CAAC,OAAO,EAAE,aAAa,GAAG,OAAO,CAAC,IAAI,CAAC;IAyDhD;;;;;;OAMG;YACW,WAAW;IAiBzB;;;;;;;;OAQG;YACW,KAAK;IA2BnB,0FAA0F;YAC5E,KAAK;IAanB;;;;OAIG;IACG,MAAM,CACV,OAAO,GAAE;QACP,OAAO,CAAC,EAAE,OAAO,CAAC;QAClB,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,wDAAwD;QACxD,KAAK,CAAC,EAAE,oBAAoB,CAAC,OAAO,CAAC,GAAG,IAAI,CAAC;KACzC,GACL,OAAO,CAAC,oBAAoB,GAAG,IAAI,CAAC;CAkBxC;AAED;;;;;GAKG;AACH,wBAAsB,YAAY,CAChC,MAAM,EAAE,oBAAoB,GAAG,IAAI,EACnC,QAAQ,EAAE,SAAS,aAAa,EAAE,GACjC,OAAO,CAAC,oBAAoB,GAAG,IAAI,CAAC,CAItC;AAED;;;GAGG;AACH,wBAAgB,WAAW,CACzB,MAAM,EAAE,oBAAoB,GAAG,IAAI,EACnC,OAAO,EAAE,aAAa,GACrB,OAAO,CAAC,oBAAoB,GAAG,IAAI,CAAC,CAEtC;AAED,wEAAwE;AACxE,wBAAgB,SAAS,CAAC,MAAM,EAAE,oBAAoB,EAAE,GAAG,EAAE,MAAM,GAAG,OAAO,CAG5E;AAED,+FAA+F;AAC/F,wBAAgB,aAAa,CAC3B,MAAM,EAAE,oBAAoB,EAC5B,GAAG,EAAE,MAAM,GACV,OAAO,CAIT;AAED;;;;;;;GAOG;AACH,wBAAgB,mBAAmB,CAAC,MAAM,EAAE,oBAAoB,GAAG,MAAM,EAAE,CAwB1E;AAgBD;;;GAGG;AACH,wBAAgB,wBAAwB,CACtC,KAAK,EAAE,OAAO,GACb,oBAAoB,GAAG,IAAI,CA4E7B"}