@classytic/repo-core 0.7.0 → 0.8.0

package/CHANGELOG.md

@@ -4,6 +4,17 @@ All notable changes to `@classytic/repo-core` are documented here.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.8.0] - 2026-07-08
+
+### Added — data-lifecycle contract (archive, streaming, distribution awareness)
+
+Additive release: existing kits on 0.7 semantics compile and pass conformance unchanged (new scenarios are capability-gated and default to skipped).
+
+- **`archiveByFilter` + `runChunkedArchive`** (`repository/archive.ts`) — the cold-storage twin of `purgeByField`. Hosts provide an `ArchiveSink` (archive table/collection, JSONL, warehouse loader); kits provide an `ArchivePort` (`readChunk` in stable PK order / `deleteChunk` by docs). The orchestrator enforces **write-before-delete** (a crash re-archives, never loses; sinks must be duplicate-tolerant — at-least-once by design), chunked `batchSize` (default 1000), per-step retry via the shared `RetryPolicy`, abort between chunks, cumulative progress, and phase-tagged errors (`read` / `sink` / `delete` — a `sink` failure guarantees rows are still hot). New capability flag: `archiveByFilter`.
+- **`cursor()` declared on `StandardRepo`** (+ `CursorOptions`) — the streaming-reads method mongokit and sqlitekit already implement is now part of the portable contract (`AsyncIterable<TDoc>`, internal `batchSize` fetching, non-snapshot semantics documented). Gated by the existing `streaming` capability.
+- **Distribution-key awareness** (`repository/distribution.ts`) — `DistributionConfig` (`key`, `onMissingKey: 'warn' | 'throw' | 'off'`, `exemptOperations`) + `createDistributionGuard` + `filterReferencesKey` (Filter IR via `collectFields`, raw records incl. `$and`/`$or`/`AND`/`OR` branches). Catches shard/partition-key-missing scatter-gather queries at the access layer; deliberately does NOT implement routing/rebalancing/partition DDL — those stay database-native (Mongo sharding, pg_partman/Timescale).
+- **Conformance**: new capability-gated scenario groups — `archiveByFilter` (sink round-trip, write-before-delete on sink failure, chunking + progress, idempotent re-run, abort partial) and `cursor` (exact-once iteration across batch boundaries, early-break safety).
+
 ## [0.7.0] - 2026-07-04
 
 ### Added — `./sync` change-log / cursor contract

package/README.md

@@ -232,6 +232,8 @@ Consumed by:
 
 See [INFRA.md](./INFRA.md) for the architectural principles, subpath map, build/tooling decisions, and the roadmap for pgkit / prismakit.
 
+See [docs/data-lifecycle.md](./docs/data-lifecycle.md) for the billion-row runbook — retention/TTL, `archiveByFilter` cold-storage extraction, `cursor()` streaming scans, tenant purge, the distribution-key guard, and the per-backend sharding/partitioning/backup playbook (what the kits own vs what the database owns).
+
 ## License
 
 MIT — see [LICENSE](./LICENSE).

package/dist/filter/index.mjs

@@ -1,7 +1,7 @@
+import { isFilter } from "./guard.mjs";
+import { collectFields, mapFilter, walkFilter } from "./walk.mjs";
 import { FALSE, TRUE, and, between, contains, endsWith, eq, exists, gt, gte, iEq, in_, isNotNull, isNull, like, lt, lte, ne, nin, not, or, raw, regex, startsWith } from "./builders.mjs";
 import { recordToFilter } from "./from-record.mjs";
-import { isFilter } from "./guard.mjs";
 import { asPredicate, matchFilter } from "./match.mjs";
 import { SCOPE_ANY, buildTenantScope, mergeScope } from "./scope.mjs";
-import { collectFields, mapFilter, walkFilter } from "./walk.mjs";
 export { FALSE, SCOPE_ANY, TRUE, and, in_ as anyOf, asPredicate, between, buildTenantScope, collectFields, contains, endsWith, eq, exists, gt, gte, iEq, in_, not as invert, isFilter, isNotNull, isNull, like, lt, lte, mapFilter, matchFilter, mergeScope, ne, nin, nin as noneOf, not, or, raw, recordToFilter, regex, startsWith, walkFilter };

package/dist/filter/scope.mjs

@@ -1,5 +1,5 @@
-import { TRUE, and, eq } from "./builders.mjs";
 import { isFilter } from "./guard.mjs";
+import { TRUE, and, eq } from "./builders.mjs";
 //#region src/filter/scope.ts
 /**
 * Scope-injection helpers.

package/dist/repository/archive.d.mts

@@ -0,0 +1,107 @@
+import { RetryPolicy } from "./resilience.mjs";
+
+//#region src/repository/archive.d.ts
+/**
+ * Destination for archived documents — implemented by the HOST, not the
+ * kit. One method so anything writable fits: an archive collection/table,
+ * an object-store JSONL writer, a warehouse ingestion API.
+ *
+ * **Idempotency requirement.** `write` MAY receive the same chunk more
+ * than once (crash between write and delete, or a chunk-level retry).
+ * Implementations must tolerate duplicates: upsert by primary key when the
+ * destination is a table/collection; dedup on load when it's a file/queue.
+ */
+interface ArchiveSink<TDoc = unknown> {
+  /**
+   * Persist one chunk. Throwing aborts the run BEFORE the chunk is
+   * deleted from the hot store — the failed chunk stays hot, nothing is
+   * lost. Retries (when configured) wrap this call.
+   */
+  write(docs: readonly TDoc[]): Promise<void>;
+  /**
+   * Optional finalize hook — called once after the run completes without
+   * error (flush buffers, close multipart uploads, commit manifests).
+   * NOT called on abort/error; partial sink output must already be safe
+   * by construction (see idempotency requirement).
+   */
+  flush?(): Promise<void>;
+}
+/**
+ * Driver-facing port the orchestrator drives. Each kit implements one
+ * closure over its driver primitives + the archive predicate.
+ *
+ * **Plugin-bypass invariant.** Like `PurgePort`, implementations MUST
+ * bypass tenant-scope injection on inner reads/deletes — the caller's
+ * filter IS the authoritative predicate.
+ */
+interface ArchivePort<TDoc = unknown> {
+  /**
+   * Read the next chunk of matching rows in a stable order (primary-key
+   * ascending), at most `limit`. Returning `[]` signals "no more matching
+   * rows"; a partial chunk (`< limit`) is also terminal after processing.
+   */
+  readChunk(limit: number): Promise<readonly TDoc[]>;
+  /**
+   * Remove the given (already-sunk) docs from the hot store. Returns the
+   * count actually deleted — may be `< docs.length` when a concurrent
+   * writer already removed some; the orchestrator reports what the port
+   * returns.
+   */
+  deleteChunk(docs: readonly TDoc[]): Promise<number>;
+}
+/** Per-call options for `archiveByFilter`. Mirrors `TenantPurgeOptions`. */
+interface ArchiveOptions {
+  /** Rows per chunk. Default 1000. */
+  batchSize?: number;
+  /** Per-chunk progress callback. `processed` is cumulative deleted count. */
+  onProgress?: (event: ArchiveProgress) => void | Promise<void>;
+  /**
+   * Abort signal. Checked between chunks — never mid-chunk. Chunks already
+   * written + deleted stay archived (at-least-once semantics).
+   */
+  signal?: AbortSignal;
+  /**
+   * Retry transient failures at the STEP level (read, sink write, delete
+   * each retry independently). Default: no retry — first error aborts.
+   */
+  retry?: RetryPolicy;
+}
+/** Chunk-level progress event. */
+interface ArchiveProgress {
+  /** Rows fully archived so far (written to sink AND deleted). */
+  processed: number;
+  /** Rows in the chunk that just completed. */
+  chunkSize: number;
+  /** Wall-clock ms elapsed since the call started. */
+  elapsedMs: number;
+}
+/** Final result of an `archiveByFilter` invocation. */
+interface ArchiveResult {
+  /** Rows fully archived (sunk + removed from the hot store). */
+  processed: number;
+  /** True iff the run completed without abort / error. */
+  ok: boolean;
+  /** Wall-clock ms. */
+  durationMs: number;
+  /**
+   * First error if `ok: false`. `phase` says which step failed — a
+   * `'sink'` failure guarantees the hot store still holds the chunk.
+   */
+  error?: {
+    message: string;
+    phase: 'read' | 'sink' | 'delete';
+    processed: number;
+  };
+}
+/**
+ * Drive a chunked archive to completion. Returns an `ArchiveResult`
+ * envelope — never throws for in-run errors (those wrap into
+ * `result.error`); only throws for invalid input (`batchSize < 1`).
+ *
+ * @param options Chunking + signal + progress + optional retry.
+ * @param sink    Host-provided destination (idempotent writes).
+ * @param port    Kit-specific driver glue (readChunk / deleteChunk).
+ */
+declare function runChunkedArchive<TDoc = unknown>(options: ArchiveOptions, sink: ArchiveSink<TDoc>, port: ArchivePort<TDoc>): Promise<ArchiveResult>;
+//#endregion
+export { ArchiveOptions, ArchivePort, ArchiveProgress, ArchiveResult, ArchiveSink, runChunkedArchive };

package/dist/repository/archive.mjs

@@ -0,0 +1,64 @@
+import { withRetry } from "./resilience.mjs";
+//#region src/repository/archive.ts
+/**
+* Drive a chunked archive to completion. Returns an `ArchiveResult`
+* envelope — never throws for in-run errors (those wrap into
+* `result.error`); only throws for invalid input (`batchSize < 1`).
+*
+* @param options Chunking + signal + progress + optional retry.
+* @param sink    Host-provided destination (idempotent writes).
+* @param port    Kit-specific driver glue (readChunk / deleteChunk).
+*/
+async function runChunkedArchive(options, sink, port) {
+	const start = Date.now();
+	const batchSize = options.batchSize ?? 1e3;
+	if (!Number.isInteger(batchSize) || batchSize < 1) throw new Error("archiveByFilter: batchSize must be a positive integer");
+	const retry = options.retry;
+	let processed = 0;
+	let phase = "read";
+	try {
+		while (true) {
+			if (options.signal?.aborted) return {
+				processed,
+				ok: false,
+				durationMs: Date.now() - start
+			};
+			phase = "read";
+			const docs = await withRetry(() => port.readChunk(batchSize), retry, options.signal);
+			if (docs.length === 0) break;
+			phase = "sink";
+			await withRetry(() => sink.write(docs), retry, options.signal);
+			phase = "delete";
+			const deleted = await withRetry(() => port.deleteChunk(docs), retry, options.signal);
+			processed += deleted;
+			if (options.onProgress) await options.onProgress({
+				processed,
+				chunkSize: deleted,
+				elapsedMs: Date.now() - start
+			});
+			if (docs.length < batchSize) break;
+		}
+		if (sink.flush) {
+			phase = "sink";
+			await withRetry(() => sink.flush?.() ?? Promise.resolve(), retry, options.signal);
+		}
+	} catch (err) {
+		return {
+			processed,
+			ok: false,
+			durationMs: Date.now() - start,
+			error: {
+				message: err instanceof Error ? err.message : String(err),
+				phase,
+				processed
+			}
+		};
+	}
+	return {
+		processed,
+		ok: true,
+		durationMs: Date.now() - start
+	};
+}
+//#endregion
+export { runChunkedArchive };

package/dist/repository/capabilities.d.mts

@@ -126,6 +126,12 @@ interface RepoCapabilities {
    * tenant cleanup primitive.
    */
   purgeByField?: boolean;
+  /**
+   * `archiveByFilter(filter, sink, options)` — chunked cold-storage
+   * extraction (write-before-delete, at-least-once). The data-lifecycle
+   * twin of `purgeByField`.
+   */
+  archiveByFilter?: boolean;
   /**
    * Mongo-style array update operators (`$push`, `$pull`, `$addToSet`,
    * `$pop`, `$pullAll`). Mongokit: native. Sqlitekit: implemented over

package/dist/repository/distribution.d.mts

@@ -0,0 +1,48 @@
+import { FilterInput } from "./types.mjs";
+
+//#region src/repository/distribution.d.ts
+/** Per-repo declaration of how the underlying table/collection is distributed. */
+interface DistributionConfig {
+  /**
+   * The shard key (Mongo) / partition key (Postgres, Timescale) / primary
+   * access dimension. Filters that omit this field fan out across every
+   * shard/partition.
+   */
+  key: string;
+  /**
+   * What to do when a filter misses the key:
+   *   - `'warn'`  (default) — invoke `onMiss` (kits default it to a
+   *     once-per-operation console warning outside production).
+   *   - `'throw'` — reject the operation; for hosts where scatter-gather
+   *     is never acceptable.
+   *   - `'off'`   — declaration only (still surfaced via metadata).
+   */
+  onMissingKey?: 'warn' | 'throw' | 'off';
+  /**
+   * Operations exempt from the check. `getById` and other primary-key
+   * lookups never carry the filter, so kits only guard the filter-taking
+   * verbs; list verbs a host legitimately runs cross-shard here
+   * (e.g. `'aggregate'` for global dashboards).
+   */
+  exemptOperations?: readonly string[];
+}
+/**
+ * Returns true when `filter` references the distribution key anywhere in
+ * the tree (Filter IR or raw record, including `$and`/`$or`/`AND`/`OR`
+ * branches). A key inside an `$or` still bounds the fan-out on Mongo and
+ * prunes partitions on Postgres, so it counts.
+ */
+declare function filterReferencesKey(filter: FilterInput | undefined, key: string): boolean;
+/** Callback invoked when a guarded operation misses the distribution key. */
+type DistributionMissHandler = (info: {
+  operation: string;
+  key: string;
+}) => void;
+/**
+ * Build a checker kits call at the top of each filter-taking verb.
+ * Stateless and allocation-free on the hit path; `onMiss` fires at most
+ * once per operation name per guard instance to keep logs readable.
+ */
+declare function createDistributionGuard(config: DistributionConfig, onMiss?: DistributionMissHandler): (operation: string, filter: FilterInput | undefined) => void;
+//#endregion
+export { DistributionConfig, DistributionMissHandler, createDistributionGuard, filterReferencesKey };

package/dist/repository/distribution.mjs

@@ -0,0 +1,73 @@
+import { isFilter } from "../filter/guard.mjs";
+import { collectFields } from "../filter/walk.mjs";
+//#region src/repository/distribution.ts
+/**
+* Distribution-key awareness — the lean, industry-standard slice of
+* "sharding support" that belongs in an access layer.
+*
+* Sharding and partitioning themselves are DATABASE features (Mongo
+* `sh.shardCollection`, Postgres declarative partitioning / pg_partman /
+* TimescaleDB hypertables) — repo-core deliberately does NOT reimplement
+* routing, rebalancing, or partition DDL. What the access layer CAN do is
+* catch the classic production regression: a query that omits the
+* shard/partition key and silently degrades to a scatter-gather (Mongo)
+* or full-partition scan (Postgres) at 100× the cost.
+*
+* A kit (or host) declares the repo's distribution key once; the guard
+* inspects each filter and reports whether the key is present. Kits wire
+* it as a dev-time warning (default) or a hard throw for strict hosts.
+*
+* Multi-tenant note: when the tenant field IS the shard key (the common
+* design — tenant-prefixed shard keys / tenant-hash partitions), kits with
+* tenant-scope injection already guarantee the key on every query; the
+* guard then only fires on `bypassTenant` escape hatches — exactly the
+* calls that deserve scrutiny.
+*/
+/**
+* Returns true when `filter` references the distribution key anywhere in
+* the tree (Filter IR or raw record, including `$and`/`$or`/`AND`/`OR`
+* branches). A key inside an `$or` still bounds the fan-out on Mongo and
+* prunes partitions on Postgres, so it counts.
+*/
+function filterReferencesKey(filter, key) {
+	if (!filter) return false;
+	if (isFilter(filter)) return collectFields(filter).includes(key);
+	return recordReferencesKey(filter, key);
+}
+function recordReferencesKey(record, key) {
+	for (const [field, value] of Object.entries(record)) {
+		if (field === key) return true;
+		if (/^(\$and|\$or|\$nor|AND|OR)$/.test(field) && Array.isArray(value)) {
+			if (value.some((child) => recordReferencesKey(child, key))) return true;
+			continue;
+		}
+		if (/^(\$not|NOT)$/.test(field) && value && typeof value === "object") {
+			if (recordReferencesKey(value, key)) return true;
+		}
+	}
+	return false;
+}
+/**
+* Build a checker kits call at the top of each filter-taking verb.
+* Stateless and allocation-free on the hit path; `onMiss` fires at most
+* once per operation name per guard instance to keep logs readable.
+*/
+function createDistributionGuard(config, onMiss) {
+	const mode = config.onMissingKey ?? "warn";
+	const exempt = new Set(config.exemptOperations ?? []);
+	const warned = /* @__PURE__ */ new Set();
+	return (operation, filter) => {
+		if (mode === "off" || exempt.has(operation)) return;
+		if (filterReferencesKey(filter, config.key)) return;
+		if (mode === "throw") throw new Error(`Distribution guard: ${operation} filter omits the distribution key "${config.key}" — this fans out across every shard/partition. Include the key, exempt the operation, or set onMissingKey: 'off'.`);
+		if (!warned.has(operation)) {
+			warned.add(operation);
+			onMiss?.({
+				operation,
+				key: config.key
+			});
+		}
+	};
+}
+//#endregion
+export { createDistributionGuard, filterReferencesKey };

package/dist/repository/index.d.mts

@@ -1,11 +1,13 @@
 import { LookupPopulateOptions, LookupPopulateResult, LookupRow, LookupSpec } from "../lookup/types.mjs";
 import { UpdateInput } from "../update/types.mjs";
 import { nestDottedKeys, nestDottedKeysAll } from "./agg-output.mjs";
+import { RetryPolicy, throwIfAborted, withRetry } from "./resilience.mjs";
+import { ArchiveOptions, ArchivePort, ArchiveProgress, ArchiveResult, ArchiveSink, runChunkedArchive } from "./archive.mjs";
 import { PLUGIN_ORDER_CONSTRAINTS, Plugin, PluginFunction, PluginType, validatePluginOrder } from "./plugin-types.mjs";
 import { RepositoryBase, RepositoryBaseOptions } from "./base.mjs";
 import { AggregateOpsSupport, RepoCapabilities } from "./capabilities.mjs";
+import { AggCacheOptions, AggDateBucket, AggDateBucketInterval, AggDateBucketUnit, AggExecutionHints, AggMeasure, AggPaginationRequest, AggRequest, AggResult, AggRow, AggTopN, AggTopNTies, BulkCreateResult, BulkWriteOperation, BulkWriteResult, ChangeEvent, ClaimTransition, ClaimVersionTransition, CursorOptions, DeleteManyResult, DeleteOptions, DeleteResult, FilterInput, FindAllOptions, FindOneAndUpdateOptions, InferDoc, KeysetAggPaginationResult, MinimalRepo, PaginationParams, QueryOptions, RepositorySession, StandardRepo, TenantPurgeOptions, TenantPurgeProgress, TenantPurgeResult, TenantPurgeStrategy, UpdateManyResult, WatchOptions, WriteOptions } from "./types.mjs";
+import { DistributionConfig, DistributionMissHandler, createDistributionGuard, filterReferencesKey } from "./distribution.mjs";
 import { STANDARD_REPO_OPTION_KEYS, StandardRepoOptionKey } from "./options.mjs";
-import { RetryPolicy, throwIfAborted, withRetry } from "./resilience.mjs";
-import { AggCacheOptions, AggDateBucket, AggDateBucketInterval, AggDateBucketUnit, AggExecutionHints, AggMeasure, AggPaginationRequest, AggRequest, AggResult, AggRow, AggTopN, AggTopNTies, BulkCreateResult, BulkWriteOperation, BulkWriteResult, ChangeEvent, ClaimTransition, ClaimVersionTransition, DeleteManyResult, DeleteOptions, DeleteResult, FilterInput, FindAllOptions, FindOneAndUpdateOptions, InferDoc, KeysetAggPaginationResult, MinimalRepo, PaginationParams, QueryOptions, RepositorySession, StandardRepo, TenantPurgeOptions, TenantPurgeProgress, TenantPurgeResult, TenantPurgeStrategy, UpdateManyResult, WatchOptions, WriteOptions } from "./types.mjs";
 import { PurgePort, WritingPurgeStrategy, runChunkedPurge } from "./purge.mjs";
-export { type AggCacheOptions, type AggDateBucket, type AggDateBucketInterval, type AggDateBucketUnit, type AggExecutionHints, type AggMeasure, type AggPaginationRequest, type AggRequest, type AggResult, type AggRow, type AggTopN, type AggTopNTies, type AggregateOpsSupport, type BulkCreateResult, type BulkWriteOperation, type BulkWriteResult, type ChangeEvent, type ClaimTransition, type ClaimVersionTransition, type DeleteManyResult, type DeleteOptions, type DeleteResult, type FilterInput, type FindAllOptions, type FindOneAndUpdateOptions, type InferDoc, type KeysetAggPaginationResult, type LookupPopulateOptions, type LookupPopulateResult, type LookupRow, type LookupSpec, type MinimalRepo, PLUGIN_ORDER_CONSTRAINTS, type PaginationParams, type Plugin, type PluginFunction, type PluginType, type PurgePort, type QueryOptions, type RepoCapabilities, RepositoryBase, type RepositoryBaseOptions, type RepositorySession, type RetryPolicy, STANDARD_REPO_OPTION_KEYS, type StandardRepo, type StandardRepoOptionKey, type TenantPurgeOptions, type TenantPurgeProgress, type TenantPurgeResult, type TenantPurgeStrategy, type UpdateInput, type UpdateManyResult, type WatchOptions, type WriteOptions, type WritingPurgeStrategy, nestDottedKeys, nestDottedKeysAll, runChunkedPurge, throwIfAborted, validatePluginOrder, withRetry };
+export { type AggCacheOptions, type AggDateBucket, type AggDateBucketInterval, type AggDateBucketUnit, type AggExecutionHints, type AggMeasure, type AggPaginationRequest, type AggRequest, type AggResult, type AggRow, type AggTopN, type AggTopNTies, type AggregateOpsSupport, type ArchiveOptions, type ArchivePort, type ArchiveProgress, type ArchiveResult, type ArchiveSink, type BulkCreateResult, type BulkWriteOperation, type BulkWriteResult, type ChangeEvent, type ClaimTransition, type ClaimVersionTransition, type CursorOptions, type DeleteManyResult, type DeleteOptions, type DeleteResult, type DistributionConfig, type DistributionMissHandler, type FilterInput, type FindAllOptions, type FindOneAndUpdateOptions, type InferDoc, type KeysetAggPaginationResult, type LookupPopulateOptions, type LookupPopulateResult, type LookupRow, type LookupSpec, type MinimalRepo, PLUGIN_ORDER_CONSTRAINTS, type PaginationParams, type Plugin, type PluginFunction, type PluginType, type PurgePort, type QueryOptions, type RepoCapabilities, RepositoryBase, type RepositoryBaseOptions, type RepositorySession, type RetryPolicy, STANDARD_REPO_OPTION_KEYS, type StandardRepo, type StandardRepoOptionKey, type TenantPurgeOptions, type TenantPurgeProgress, type TenantPurgeResult, type TenantPurgeStrategy, type UpdateInput, type UpdateManyResult, type WatchOptions, type WriteOptions, type WritingPurgeStrategy, createDistributionGuard, filterReferencesKey, nestDottedKeys, nestDottedKeysAll, runChunkedArchive, runChunkedPurge, throwIfAborted, validatePluginOrder, withRetry };

package/dist/repository/index.mjs

@@ -1,7 +1,9 @@
 import { nestDottedKeys, nestDottedKeysAll } from "./agg-output.mjs";
+import { throwIfAborted, withRetry } from "./resilience.mjs";
+import { runChunkedArchive } from "./archive.mjs";
 import { PLUGIN_ORDER_CONSTRAINTS, validatePluginOrder } from "./plugin-types.mjs";
 import { RepositoryBase } from "./base.mjs";
+import { createDistributionGuard, filterReferencesKey } from "./distribution.mjs";
 import { STANDARD_REPO_OPTION_KEYS } from "./options.mjs";
-import { throwIfAborted, withRetry } from "./resilience.mjs";
 import { runChunkedPurge } from "./purge.mjs";
-export { PLUGIN_ORDER_CONSTRAINTS, RepositoryBase, STANDARD_REPO_OPTION_KEYS, nestDottedKeys, nestDottedKeysAll, runChunkedPurge, throwIfAborted, validatePluginOrder, withRetry };
+export { PLUGIN_ORDER_CONSTRAINTS, RepositoryBase, STANDARD_REPO_OPTION_KEYS, createDistributionGuard, filterReferencesKey, nestDottedKeys, nestDottedKeysAll, runChunkedArchive, runChunkedPurge, throwIfAborted, validatePluginOrder, withRetry };

package/dist/repository/types.d.mts

@@ -2,9 +2,10 @@ import { Filter } from "../filter/types.mjs";
 import { OffsetPaginationResult } from "../pagination/types.mjs";
 import { LookupPopulateOptions, LookupPopulateResult, LookupSpec } from "../lookup/types.mjs";
 import { UpdateInput } from "../update/types.mjs";
+import { RetryPolicy } from "./resilience.mjs";
+import { ArchiveOptions, ArchiveResult, ArchiveSink } from "./archive.mjs";
 import { RepoCapabilities } from "./capabilities.mjs";
 import { CacheOptions } from "../cache/options.mjs";
-import { RetryPolicy } from "./resilience.mjs";
 
 //#region src/repository/types.d.ts
 /**
@@ -1525,6 +1526,46 @@ interface StandardRepo<TDoc> extends MinimalRepo<TDoc> {
    * @param options  Chunking, session, progress, abort signal.
    */
   purgeByField?(field: string, value: unknown, strategy: TenantPurgeStrategy, options?: TenantPurgeOptions): Promise<TenantPurgeResult>;
+  /**
+   * Chunked cold-storage extraction — move every row matching `filter`
+   * into a host-provided {@link ArchiveSink}, then remove it from the hot
+   * store. THE data-lifecycle primitive for retention windows and
+   * hot-table size control ("archive orders older than 18 months").
+   *
+   * **Write-before-delete.** Rows leave the hot store only after the sink
+   * acknowledged the chunk — a crash re-archives the same chunk (sinks
+   * must be duplicate-tolerant); data is never lost. At-least-once, the
+   * same envelope semantics as `purgeByField`.
+   *
+   * **Index requirement.** Same as purge: the filter's leading field(s)
+   * must be indexed or every chunk re-scans the table.
+   *
+   * **When NOT to use it.** On Postgres with time-partitioned tables,
+   * detaching/dropping a partition archives a billion rows in O(1) —
+   * prefer that (pg_partman / Timescale retention policies) and keep this
+   * method for non-partitioned tables and portable code paths.
+   *
+   * Optional method; gate on `capabilities.archiveByFilter`.
+   *
+   * @param filter  Predicate selecting rows to archive.
+   * @param sink    Destination (archive table/collection, JSONL, ...).
+   * @param options Chunking, progress, abort signal, retry.
+   */
+  archiveByFilter?(filter: FilterInput, sink: ArchiveSink<TDoc>, options?: ArchiveOptions): Promise<ArchiveResult>;
+  /**
+   * Streaming reads — an async iterator over every row matching `filter`,
+   * fetched in `batchSize` chunks (keyset-progressed on SQL kits, native
+   * driver cursor on Mongo) so billion-row scans never load the result
+   * set into memory. Drive with `for await`; breaking out releases the
+   * underlying cursor/batch loop.
+   *
+   * Rows inserted behind the iteration point during the scan are not
+   * revisited; rows inserted ahead may appear — the usual non-snapshot
+   * cursor semantics every backend shares.
+   *
+   * Optional method; gate on `capabilities.streaming`.
+   */
+  cursor?(filter?: FilterInput, options?: CursorOptions): AsyncIterable<TDoc>;
   /**
    * Heterogeneous bulk write. Stays optional — kits dispatch each op
    * against the appropriate driver primitive inside a single transaction;
@@ -1668,5 +1709,17 @@ interface WatchOptions {
    */
   resumeAfter?: unknown;
 }
+/**
+ * Options for `StandardRepo.cursor()` streaming reads. The open index
+ * signature lets kit-specific knobs (mongokit's `organizationId` opt-out,
+ * read-preference hints) flow without widening the portable contract.
+ */
+interface CursorOptions {
+  /** Rows fetched per underlying batch. Default kit-specific (~100–1000). */
+  batchSize?: number;
+  /** Iteration order. Kits default to primary-key ascending. */
+  sort?: Record<string, 1 | -1>;
+  [key: string]: unknown;
+}
 //#endregion
-export { AggCacheOptions, AggDateBucket, AggDateBucketInterval, AggDateBucketUnit, AggExecutionHints, AggMeasure, AggPaginationRequest, AggRequest, AggResult, AggRow, AggTopN, AggTopNTies, BulkCreateResult, BulkWriteOperation, BulkWriteResult, ChangeEvent, ClaimTransition, ClaimVersionTransition, DeleteManyResult, DeleteOptions, DeleteResult, FilterInput, FindAllOptions, FindOneAndUpdateOptions, InferDoc, KeysetAggPaginationResult, MinimalRepo, PaginationParams, QueryOptions, RepositorySession, StandardRepo, TenantPurgeOptions, TenantPurgeProgress, TenantPurgeResult, TenantPurgeStrategy, UpdateManyResult, WatchOptions, WriteOptions };
+export { AggCacheOptions, AggDateBucket, AggDateBucketInterval, AggDateBucketUnit, AggExecutionHints, AggMeasure, AggPaginationRequest, AggRequest, AggResult, AggRow, AggTopN, AggTopNTies, BulkCreateResult, BulkWriteOperation, BulkWriteResult, ChangeEvent, ClaimTransition, ClaimVersionTransition, CursorOptions, DeleteManyResult, DeleteOptions, DeleteResult, FilterInput, FindAllOptions, FindOneAndUpdateOptions, InferDoc, KeysetAggPaginationResult, MinimalRepo, PaginationParams, QueryOptions, RepositorySession, StandardRepo, TenantPurgeOptions, TenantPurgeProgress, TenantPurgeResult, TenantPurgeStrategy, UpdateManyResult, WatchOptions, WriteOptions };

package/dist/sync/index.d.mts

@@ -38,7 +38,7 @@
 /** What happened to a document. Field-level patches are deliberately out of
  *  scope — version-checked upserts + server authority beat CRDT complexity
  *  for ERP data (the Sheets/Replicache position, not the Figma one). */
-type ChangeOp = "upsert" | "delete";
+type ChangeOp = 'upsert' | 'delete';
 interface ChangeEntry<TDoc = unknown> {
   /** Which logical collection/resource this change belongs to (e.g. `pos-order`). */
   readonly scope: string;
@@ -87,7 +87,7 @@ interface PushMutation<TDoc = unknown> {
    */
   readonly mutationId: string;
 }
-type PushVerdictStatus = "applied" | "already_applied" | "conflict" | "rejected";
+type PushVerdictStatus = 'applied' | 'already_applied' | 'conflict' | 'rejected';
 interface PushVerdict<TDoc = unknown> {
   readonly mutationId: string;
   readonly status: PushVerdictStatus;
@@ -103,11 +103,11 @@ interface ChangeLogAppendOptions {
 }
 interface ChangeLogStore<TDoc = unknown> {
   /** Record a change. `cursor`/`at` are ASSIGNED by the store; callers pass the rest. */
-  append(entry: Omit<ChangeEntry<TDoc>, "cursor" | "at">, options?: ChangeLogAppendOptions): Promise<ChangeEntry<TDoc>>;
+  append(entry: Omit<ChangeEntry<TDoc>, 'cursor' | 'at'>, options?: ChangeLogAppendOptions): Promise<ChangeEntry<TDoc>>;
   /** Entries strictly AFTER `cursor` (empty string = from the beginning). */
   since(cursor: string, options?: ChangesSinceOptions): Promise<ChangesPage<TDoc>>;
   /** The current head checkpoint — what a fresh client stores after a full load. */
-  latestCursor(options?: Pick<ChangesSinceOptions, "tenantId" | "scopes">): Promise<string>;
+  latestCursor(options?: Pick<ChangesSinceOptions, 'tenantId' | 'scopes'>): Promise<string>;
   /**
    * Compact entries older than `before`, keeping per-doc latest state.
    * Returns the new HORIZON cursor: clients checkpointed before it must full-resync.
@@ -123,7 +123,7 @@ declare class CursorExpiredError extends Error {
 declare class MemoryChangeLogStore<TDoc = unknown> implements ChangeLogStore<TDoc> {
   private entries;
   private seq;
-  append(entry: Omit<ChangeEntry<TDoc>, "cursor" | "at">, _options?: ChangeLogAppendOptions): Promise<ChangeEntry<TDoc>>;
+  append(entry: Omit<ChangeEntry<TDoc>, 'cursor' | 'at'>, _options?: ChangeLogAppendOptions): Promise<ChangeEntry<TDoc>>;
   since(cursor: string, options?: ChangesSinceOptions): Promise<ChangesPage<TDoc>>;
   latestCursor(): Promise<string>;
 }

package/dist/testing/conformance.mjs

@@ -35,6 +35,8 @@ function runStandardRepoConformance(harness) {
 	const skipNoStddev = !aggGate || !ops?.stddev;
 	const skipNoCache = !aggGate || !ops?.cache;
 	const skipNoPurge = !harness.features.purgeByField;
+	const skipNoArchive = !harness.features.archiveByFilter;
+	const skipNoStreaming = !harness.features.streaming;
 	describe(`[conformance] ${harness.name}`, () => {
 		let ctx;
 		beforeEach(async () => {
@@ -1268,6 +1270,146 @@ function runStandardRepoConformance(harness) {
 				expect(result.processed).toBe(3);
 			});
 		});
+		describe("archiveByFilter (cold-storage extraction)", () => {
+			/** In-memory sink — duplicate-tolerant by keying on the id field. */
+			const makeSink = () => {
+				const byId = /* @__PURE__ */ new Map();
+				let flushes = 0;
+				return {
+					sink: {
+						write: async (docs) => {
+							for (const doc of docs) byId.set(String(doc[harness.idField]), doc);
+						},
+						flush: async () => {
+							flushes += 1;
+						}
+					},
+					get docs() {
+						return [...byId.values()];
+					},
+					get flushes() {
+						return flushes;
+					}
+				};
+			};
+			const seedTenants = async (hot, keep) => {
+				for (let i = 0; i < hot; i++) await ctx.repo.create(harness.makeDoc({
+					name: `arch-${i}`,
+					category: "org-cold"
+				}));
+				for (let i = 0; i < keep; i++) await ctx.repo.create(harness.makeDoc({
+					name: `keep-${i}`,
+					category: "org-hot"
+				}));
+			};
+			it.skipIf(skipNoArchive)("moves matching rows into the sink and removes them from the hot store", async () => {
+				if (!ctx.repo.archiveByFilter) return;
+				await seedTenants(3, 2);
+				const memory = makeSink();
+				const result = await ctx.repo.archiveByFilter({ category: "org-cold" }, memory.sink);
+				expect(result.ok).toBe(true);
+				expect(result.processed).toBe(3);
+				expect(memory.docs).toHaveLength(3);
+				expect(await ctx.repo.count({ category: "org-cold" })).toBe(0);
+				expect(await ctx.repo.count({ category: "org-hot" })).toBe(2);
+				expect(memory.flushes).toBe(1);
+			});
+			it.skipIf(skipNoArchive)("write-before-delete: a failing sink aborts with every row still hot", async () => {
+				if (!ctx.repo.archiveByFilter) return;
+				await seedTenants(3, 0);
+				const result = await ctx.repo.archiveByFilter({ category: "org-cold" }, { write: async () => {
+					throw new Error("sink unavailable");
+				} });
+				expect(result.ok).toBe(false);
+				expect(result.error?.phase).toBe("sink");
+				expect(result.processed).toBe(0);
+				expect(await ctx.repo.count({ category: "org-cold" })).toBe(3);
+			});
+			it.skipIf(skipNoArchive)("chunking: batchSize honored, cumulative progress, all rows sunk", async () => {
+				if (!ctx.repo.archiveByFilter) return;
+				await seedTenants(25, 0);
+				const memory = makeSink();
+				const events = [];
+				expect((await ctx.repo.archiveByFilter({ category: "org-cold" }, memory.sink, {
+					batchSize: 10,
+					onProgress: (event) => {
+						events.push({
+							processed: event.processed,
+							chunkSize: event.chunkSize
+						});
+					}
+				})).processed).toBe(25);
+				expect(memory.docs).toHaveLength(25);
+				expect(events).toEqual([
+					{
+						processed: 10,
+						chunkSize: 10
+					},
+					{
+						processed: 20,
+						chunkSize: 10
+					},
+					{
+						processed: 25,
+						chunkSize: 5
+					}
+				]);
+			});
+			it.skipIf(skipNoArchive)("idempotent: a second run archives 0", async () => {
+				if (!ctx.repo.archiveByFilter) return;
+				await seedTenants(3, 0);
+				const memory = makeSink();
+				expect((await ctx.repo.archiveByFilter({ category: "org-cold" }, memory.sink)).processed).toBe(3);
+				const second = await ctx.repo.archiveByFilter({ category: "org-cold" }, memory.sink);
+				expect(second.ok).toBe(true);
+				expect(second.processed).toBe(0);
+			});
+			it.skipIf(skipNoArchive)("abort signal stops between chunks; archived chunks stay archived", async () => {
+				if (!ctx.repo.archiveByFilter) return;
+				await seedTenants(25, 0);
+				const memory = makeSink();
+				const controller = new AbortController();
+				const result = await ctx.repo.archiveByFilter({ category: "org-cold" }, memory.sink, {
+					batchSize: 10,
+					signal: controller.signal,
+					onProgress: (event) => {
+						if (event.processed === 10) controller.abort();
+					}
+				});
+				expect(result.ok).toBe(false);
+				expect(result.processed).toBe(10);
+				expect(memory.docs).toHaveLength(10);
+				expect(await ctx.repo.count({ category: "org-cold" })).toBe(15);
+			});
+		});
+		describe("cursor (streaming reads)", () => {
+			it.skipIf(skipNoStreaming)("iterates every matching row exactly once across batch boundaries", async () => {
+				if (!ctx.repo.cursor) return;
+				for (let i = 0; i < 7; i++) await ctx.repo.create(harness.makeDoc({
+					name: `stream-${i}`,
+					category: "streamable"
+				}));
+				await ctx.repo.create(harness.makeDoc({
+					name: "other",
+					category: "not-streamable"
+				}));
+				const seen = [];
+				for await (const doc of ctx.repo.cursor({ category: "streamable" }, { batchSize: 3 })) seen.push(String(doc[harness.idField]));
+				expect(seen).toHaveLength(7);
+				expect(new Set(seen).size).toBe(7);
+			});
+			it.skipIf(skipNoStreaming)("breaking out early releases the iterator cleanly", async () => {
+				if (!ctx.repo.cursor) return;
+				for (let i = 0; i < 5; i++) await ctx.repo.create(harness.makeDoc({ name: `brk-${i}` }));
+				let taken = 0;
+				for await (const _doc of ctx.repo.cursor({}, { batchSize: 2 })) {
+					taken += 1;
+					if (taken === 2) break;
+				}
+				expect(taken).toBe(2);
+				expect(await ctx.repo.count({})).toBe(5);
+			});
+		});
 	});
 }
 //#endregion

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@classytic/repo-core",
-  "version": "0.7.0",
+  "version": "0.8.0",
   "description": "Driver-agnostic repository primitives: hooks, Filter IR, operations, pagination, cache contract. Foundation for mongokit, sqlitekit, pgkit, and prismakit. Lean by design — no plugins ship here; each kit owns its own.",
   "type": "module",
   "sideEffects": false,