@lunora/do 1.0.0-alpha.31 → 1.0.0-alpha.32

package/dist/index.d.mts

@@ -2407,6 +2407,11 @@ interface MaterializeResult {
   /** `id → projected-value JSON` — the post-tick membership, to feed back as `baseline` next tick. */
   nextBaseline: Map<string, string>;
 }
+/** The outcome of one incremental materialize pass. No baseline: incremental applies only the pulled slice, never a full-membership diff. */
+interface IncrementalMaterializeResult {
+  /** Number of `CdcChange`s applied (upserts + tombstone deletes) for the pulled slice. */
+  applied: number;
+}
 /**
 * Diff `pulled` against `baseline` and apply the delta to `writer`. Returns the
 * applied count and the next baseline. A steady-state tick (membership unchanged)
@@ -2417,6 +2422,36 @@ declare const materializeExternalRows: (writer: DatabaseWriterLike, pulled: Read
   table: string;
 }) => Promise<MaterializeResult>;
 /**
+* Apply an **incremental** slice (plan 136): the freshly-pulled rows changed since
+* the watermark, upsert-only. Unlike {@link materializeExternalRows} this reads no
+* baseline and never diffs the full membership — an absent row means "unchanged
+* since the watermark", NOT "deleted". Each pulled row is projected with the SAME
+* {@link projectExternalSourceRow} full-pull uses, so an incrementally-upserted row
+* is byte-identical to how the next reconcile sweep's full-pull would store it (no
+* spurious update on reconcile).
+*
+* Delete visibility comes from `deletedIds` — the ids the caller resolved from the
+* source's soft-delete tombstone column. Every other pulled row is an `insert`,
+* which {@link applyCdcChanges} upserts (insert, or replace on conflict), so a
+* changed existing row is updated and a genuinely new row is inserted without the
+* caller tracking which is which.
+*
+* **Content short-circuit.** An incremental cursor query uses `>= watermark` (so
+* rows sharing the boundary value are never skipped), which means a steady-state
+* tick re-pulls the boundary row(s) unchanged. Blindly upserting them would append
+* a `__cdc_log` entry, broadcast a spurious `update` to every `defineShape`
+* subscriber, re-run search/aggregate/rank sync, and fire `onWrite` (a Vectorize
+* re-embed = real cost) on every tick — and `replace` would reset `_creationTime`.
+* So each row is diffed against its stored projection (the SAME
+* {@link projectExternalSourceRow} + {@link stableStringify} full-pull uses) and
+* skipped when byte-identical — mirroring the full-pull diff's steady-state no-op.
+*/
+declare const materializeExternalRowsIncremental: (writer: DatabaseWriterLike, pulled: ReadonlyArray<Record<string, unknown>>, options: {
+  columns?: ReadonlyArray<string>;
+  deletedIds?: ReadonlySet<string>;
+  table: string;
+}) => Promise<IncrementalMaterializeResult>;
+/**
 * Read the materialized table's current membership as the canonical full-pull
 * baseline (`id → canonical JSON`). Reuses the shape scanner (`selectShapeRows`)
 * and the SAME {@link projectExternalSourceRow} + {@link stableStringify} the diff
@@ -2445,14 +2480,23 @@ interface SourceClientLike {
 type SourceRefresh = "manual" | {
   everyMs: number;
 };
+/** The incremental cursor config (plan 136): the watermark column + the watermark-parameterized pull query. */
+interface SourceCursorLike {
+  column: string;
+  query: string;
+}
 /** The runtime `.source(...)` config the poll loop reads — a structural mirror of `@lunora/server`'s `ExternalSourceDefinition` (only the fields the tick uses). */
 interface ExternalSourceLike {
   binding: string;
   columns?: ReadonlyArray<string>;
+  cursor?: SourceCursorLike;
   idColumn?: string;
   map?: (row: Record<string, unknown>) => Record<string, unknown>;
+  mode?: string;
   query: string;
+  reconcileEveryMs?: number;
   refresh?: SourceRefresh;
+  softDeleteColumn?: string;
   tenantBy?: (shardKey: string) => ReadonlyArray<unknown>;
 }
 /**
@@ -2482,6 +2526,39 @@ declare const isSourceDue: (refresh: SourceRefresh | undefined, lastPolledMs: nu
 */
 declare const pullExternalSourceTick: (sql: SqlExec, writer: DatabaseWriterLike, client: SourceClientLike, table: string, source: ExternalSourceLike, shardKey: string) => Promise<MaterializeResult>;
 /**
+* Whether an upstream row is a soft-delete tombstone under `column`. A set
+* `deleted_at` (any non-null value, e.g. a timestamp), an `is_deleted = true`, or a
+* non-zero flag all read as deleted; `null` / `undefined` / `false` / `0` mean
+* live. Note an **empty string** reads as deleted (`"" !== 0`), so an upstream that
+* clears the column to `""` rather than `NULL` for a live row would mis-signal —
+* use `NULL` for live rows. The incremental query MUST return tombstoned rows
+* (don't filter `WHERE deleted_at IS NULL`) or the delete is never observed.
+*/
+declare const isSoftDeleted: (row: Record<string, unknown>, column: string) => boolean;
+/**
+* Run one **incremental** tick (plan 136). Reads the durable watermark for
+* `(table, shardKey)`; on the first ever poll or when the `reconcileEveryMs` sweep
+* is due it runs a **full-pull** (seed/GC: {@link runExternalSourceTick} observes
+* deletes and re-establishes membership), otherwise it pulls only rows past the
+* watermark via `cursor.query` and upserts them ({@link materializeExternalRowsIncremental},
+* tombstones → deletes). Either way it advances the watermark to the max cursor
+* value seen and persists it (and the reconcile timestamp).
+*
+* **Crash safety** is by ordering + idempotency, NOT an atomic transaction across
+* the two write channels (the apply goes through the `writer`; the watermark write
+* is a raw `sql` write): the watermark advances only AFTER a fully-applied slice,
+* so a crash between the apply and the watermark write leaves the watermark behind
+* and the next tick re-pulls the same `>= watermark` slice — which the upsert
+* short-circuits (unchanged rows) or re-applies idempotently. It only ever replays,
+* never skips (same self-healing argument as `advanceClientWatermark`).
+*
+* Requires `source.cursor` (validated at `defineSchema` for incremental mode); a
+* missing cursor throws rather than silently degrading to a stuck watermark.
+*/
+declare const pullExternalSourceIncrementalTick: (sql: SqlExec, writer: DatabaseWriterLike, client: SourceClientLike, table: string, source: ExternalSourceLike, shardKey: string, nowMs: number) => Promise<{
+  applied: number;
+}>;
+/**
 * Reserved `functionPath` prefix for admin introspection RPCs. These travel
 * over the same `/_lunora/rpc` → shard `/rpc` path as ordinary functions, but
 * `ShardDO` intercepts them before user dispatch and serves them from the
@@ -6704,4 +6781,4 @@ interface WhereSqlStrategy {
 * `undefined` when the input imposes no constraint (empty `where`).
 */
 declare const compileWhereSql: (where: WhereInput | undefined, strategy: WhereSqlStrategy) => SQL | undefined;
-export { ADMIN_FUNCTIONS, ADMIN_FUNCTION_PREFIX, AGGREGATE_SQL_FUNCTION, AUTH_METRICS_BUCKETS_TABLE, AUTH_METRICS_BUCKET_MS, AUTH_METRICS_BUCKET_RETENTION, AUTH_METRICS_TABLE, type AdvisoriesResult, type AdvisoryFinding, type AggregateIndexDefinitionLike, type AggregateOp, type AggregateOptions, type AggregateResult, type AggregateTally, type ApplyOnDeleteOptions, type AuditEntry, type AuditLogResult, type AuthMetrics, type AuthMetricsBucket, type BroadcastDelta, CDC_LOG_TABLE, type CacheEntry, type CapturedMailRow, type CdcChange, type Clock, type ColumnMeta, type ColumnMetaLike, ConflictError, type CountArgs, CountRlsUnsupportedError, type CtxDbOptions, DATA_MIGRATION_STATE_TABLE, DEFAULT_MAX_RELATION_KEYS, type DataMigrationDocument, type DataMigrationLike, type DataMigrationTransform, type DatabaseWriterLike, type DependencyTracker, type DeployInfo, type ExportRow, type ExportShardAdminArgs, type ExportShardArgs, type ExternalSourceDiffResult, type ExternalSourceLike, FLAGS_FUNCTION_PREFIX, FUNCTION_METRICS_BUCKETS_TABLE, FUNCTION_METRICS_BUCKET_MS, FUNCTION_METRICS_BUCKET_RETENTION, FUNCTION_METRICS_INDEX_TABLE, FUNCTION_METRICS_TABLE, type FacetColumnOptions, type FacetColumnResult, type FacetValue, type FieldOperators, type FlagEvaluation, type FlagsResult, type FunctionCallStat, type FunctionMetricBucket, type FunctionMetricIndexHit, type FunctionStatsResult, type GroupByEntry, type GroupByOptions, type HibernatableWebSocket, type IdGenerator, type ImportError, type ImportShardAdminArgs, type ImportShardArgs, type ImportShardResult, type IndexDefinitionLike, type IndexRangeBuilderLike, LogBuffer, type LogEntry, type LogEventInput, type LogLevel, type LogSink, MAIL_RETENTION, MAIL_TABLE, MAX_SQL_ROWS, MIN_ADMIN_TOKEN_LENGTH, MIN_AUTH_SECRET_LENGTH, type MaskColumnMetadata, type MaskPoliciesResult, type MaterializeResult, type MigrationDirection, type MigrationRunResult, type MigrationStatus, type MigrationStatusRow, type MutationDelta, type NestedWith, NotFoundError, NotUniqueError, type OnDeleteActionLike, type OrderByInput, type OrderKey, type PaginationOptions, type PitrBookmarkResult, type PitrRestoreArgs, type PitrRestoreResult, type PitrStorage, type QueryArgs, type QueryPage, type QueueMetadata, type QueuesResult, RANK_TIEBREAK, RELATION_FUNCTION_PREFIX, RLS_UNWRAP_SYMBOL, ROOT_DO_SIZE_WARN_BYTES, ROOT_SHARD_NAME, type RankDirection, type RankIndexDefinitionLike, type RankOptions, type RankPage, type RankPageOptions, type RankPageRow, type RankPageRowKey, type RankResult, type RankSortKeyLike, ReactiveCache, type ReactiveCacheOptions, type ReadHook, type ReadTablePageOptions, type RecordAuthEventInput, type RecordFunctionMetricInput, type RecordMailInput, type RelationDefinitionLike, type RenderedSql, type ResolveRelationPredicatesOptions, type ResolveWithOptions, type RestrictableQueryOptions, type RlsPoliciesResult, type RlsPolicyMetadata, RlsRequiredError, type RlsRoleMetadata, type RpcRequest, type RunDataMigrationOptions, type RunShardApplyCdcArgs, type RunShardApplyCdcResult, type RunShardBulkDeleteArgs, type RunShardBulkDeleteResult, type RunShardExportArgs, type RunShardImportArgs, type RunShardMigrationArgs, type RunShardRankBeforeArgs, type RunShardRankPageArgs, type RunShardWriteArgs, type RunShardWriteResult, type RunTriggersOptions, SCAN_DEP, SESSION_DO_TTL_DEFAULT, SHARD_REGISTRY_DO_NAME, type SchedulableWorkflowReferenceLike, type ScheduledFunctionDoc, type SchedulerLike, type SchemaLike, type SearchFilterBuilderLike, type SecurityAuditResult, type SecurityFinding, type SecurityFindingKind, type SecurityFindingLevel, type SelectMatchingIdsOptions, type ServerDefaultContextLike, SessionDO, type SessionRecord, type SettingEntry, type SettingKind, type SettingsResult, type ShapeSubscriptionQuery, ShardDO, type ShardDOOptions, type ShardDOState, type ShardRankPageResult, ShardRegistryDO, type SocketAttachment, type SortDirection, type SourceClientLike, type SourceRefresh, type SqlConsoleResult, type SqlCursor, type SqlEngine, type SqlExec, type StorageRuleMetadata, type StorageRulesResult, type StudioFeaturesResult, type SubscriptionEnvelope, type SubscriptionOutcome, type SubscriptionQuery, type SystemDatabaseReader, type SystemDoc, type SystemQuery, type SystemReaderOptions, type SystemReaderSchedulerLike, type SystemReaderStorageLike, type SystemTableName, type TableColumnsResult, type TableDefinitionLike, type TableIndexInfo, type TableIndexesResult, type TableInfo, type TablePage, type TableReaderLike, type TablesColumnsResult, type TransactionSqlLike, type TriggerContextLike, type TriggerDefinitionLike, type TriggerEventLike, type TriggerOpLike, type TriggerTimingLike, type ValidatorLike, type WhereInput, type WhereSqlStrategy, type WithInput, type WorkflowMetadata, type WorkflowsResult, type WriteEvent, type WriteHook, aggregateSqlFunction, aggregateTableName, applyCdcChanges, applyOnDelete, applySelect, armRestore, assertFlatPredicate, assertReadonly, assertShapeShardable, assertValidClientId, backfillAggregateIndexes, backfillRankIndexes, buildFtsMatch, buildSecurityAudit, buildSeekWhere, clearCapturedMail, coerceAggregateNumber, compileWhereSql, containsRelationPredicate, createDependencyTracker, createShardCtxDb, createSystemReader, decodeCursor, depKey, diffExternalSource, encodeAggregateKey, encodeCursor, encodePartitionKey, ensureAuthMetricsTables, ensureFunctionMetricsTables, ensureMailTable, exportShardRows, exportShardTable, facetColumn, fanOutScalarCounts, foldAggregateTally, ftsTableName, guardWriter, hasTrigger, importShardRows, isRelationPredicate, isSourceDue, liftSourceId, listTables, matchesRankStaticWhere, matchesStaticWhere, materializeExternalRows, mergeWhere, normalizeCountArgument, normalizeIdStructurally, normalizeOrderKeys, parseExportShardArgs, parseImportShardArgs, planAggregateLookup, pullExternalSourceTick, rankTableName, reactiveCacheKey, readAggregateValue, readAuthMetrics, readBookmark, readCapturedMail, readCdcChanges, readExternalSourceBaseline, readFunctionMetricBuckets, readFunctionMetricIndexHits, readFunctionMetrics, readFunctionMetricsTotals, readMigrationStatus, readTablePage, recordAuthEvent, recordCapturedMail, recordFunctionMetric, renderSql, resolveRankPartition, resolveRelationPredicates, resolveWith, runDataMigration, runExternalSourceTick, runReadonlySql, runRowValidators, runShardMigrations, runTriggers, scoreDocument, selectExportTables, selectIndexForAggregate, selectIndexForCount, selectIndexForGroupBy, selectMatchingIds, serveRelationFanout, softDeleteScope, sortColumnName, stableStringify, stableWireKey, stringifySearchText, subscriptionListDeltas, throwingScheduler, tokenizeSearch, trimCdcChanges, validateImportRow };
+export { ADMIN_FUNCTIONS, ADMIN_FUNCTION_PREFIX, AGGREGATE_SQL_FUNCTION, AUTH_METRICS_BUCKETS_TABLE, AUTH_METRICS_BUCKET_MS, AUTH_METRICS_BUCKET_RETENTION, AUTH_METRICS_TABLE, type AdvisoriesResult, type AdvisoryFinding, type AggregateIndexDefinitionLike, type AggregateOp, type AggregateOptions, type AggregateResult, type AggregateTally, type ApplyOnDeleteOptions, type AuditEntry, type AuditLogResult, type AuthMetrics, type AuthMetricsBucket, type BroadcastDelta, CDC_LOG_TABLE, type CacheEntry, type CapturedMailRow, type CdcChange, type Clock, type ColumnMeta, type ColumnMetaLike, ConflictError, type CountArgs, CountRlsUnsupportedError, type CtxDbOptions, DATA_MIGRATION_STATE_TABLE, DEFAULT_MAX_RELATION_KEYS, type DataMigrationDocument, type DataMigrationLike, type DataMigrationTransform, type DatabaseWriterLike, type DependencyTracker, type DeployInfo, type ExportRow, type ExportShardAdminArgs, type ExportShardArgs, type ExternalSourceDiffResult, type ExternalSourceLike, FLAGS_FUNCTION_PREFIX, FUNCTION_METRICS_BUCKETS_TABLE, FUNCTION_METRICS_BUCKET_MS, FUNCTION_METRICS_BUCKET_RETENTION, FUNCTION_METRICS_INDEX_TABLE, FUNCTION_METRICS_TABLE, type FacetColumnOptions, type FacetColumnResult, type FacetValue, type FieldOperators, type FlagEvaluation, type FlagsResult, type FunctionCallStat, type FunctionMetricBucket, type FunctionMetricIndexHit, type FunctionStatsResult, type GroupByEntry, type GroupByOptions, type HibernatableWebSocket, type IdGenerator, type ImportError, type ImportShardAdminArgs, type ImportShardArgs, type ImportShardResult, type IncrementalMaterializeResult, type IndexDefinitionLike, type IndexRangeBuilderLike, LogBuffer, type LogEntry, type LogEventInput, type LogLevel, type LogSink, MAIL_RETENTION, MAIL_TABLE, MAX_SQL_ROWS, MIN_ADMIN_TOKEN_LENGTH, MIN_AUTH_SECRET_LENGTH, type MaskColumnMetadata, type MaskPoliciesResult, type MaterializeResult, type MigrationDirection, type MigrationRunResult, type MigrationStatus, type MigrationStatusRow, type MutationDelta, type NestedWith, NotFoundError, NotUniqueError, type OnDeleteActionLike, type OrderByInput, type OrderKey, type PaginationOptions, type PitrBookmarkResult, type PitrRestoreArgs, type PitrRestoreResult, type PitrStorage, type QueryArgs, type QueryPage, type QueueMetadata, type QueuesResult, RANK_TIEBREAK, RELATION_FUNCTION_PREFIX, RLS_UNWRAP_SYMBOL, ROOT_DO_SIZE_WARN_BYTES, ROOT_SHARD_NAME, type RankDirection, type RankIndexDefinitionLike, type RankOptions, type RankPage, type RankPageOptions, type RankPageRow, type RankPageRowKey, type RankResult, type RankSortKeyLike, ReactiveCache, type ReactiveCacheOptions, type ReadHook, type ReadTablePageOptions, type RecordAuthEventInput, type RecordFunctionMetricInput, type RecordMailInput, type RelationDefinitionLike, type RenderedSql, type ResolveRelationPredicatesOptions, type ResolveWithOptions, type RestrictableQueryOptions, type RlsPoliciesResult, type RlsPolicyMetadata, RlsRequiredError, type RlsRoleMetadata, type RpcRequest, type RunDataMigrationOptions, type RunShardApplyCdcArgs, type RunShardApplyCdcResult, type RunShardBulkDeleteArgs, type RunShardBulkDeleteResult, type RunShardExportArgs, type RunShardImportArgs, type RunShardMigrationArgs, type RunShardRankBeforeArgs, type RunShardRankPageArgs, type RunShardWriteArgs, type RunShardWriteResult, type RunTriggersOptions, SCAN_DEP, SESSION_DO_TTL_DEFAULT, SHARD_REGISTRY_DO_NAME, type SchedulableWorkflowReferenceLike, type ScheduledFunctionDoc, type SchedulerLike, type SchemaLike, type SearchFilterBuilderLike, type SecurityAuditResult, type SecurityFinding, type SecurityFindingKind, type SecurityFindingLevel, type SelectMatchingIdsOptions, type ServerDefaultContextLike, SessionDO, type SessionRecord, type SettingEntry, type SettingKind, type SettingsResult, type ShapeSubscriptionQuery, ShardDO, type ShardDOOptions, type ShardDOState, type ShardRankPageResult, ShardRegistryDO, type SocketAttachment, type SortDirection, type SourceClientLike, type SourceCursorLike, type SourceRefresh, type SqlConsoleResult, type SqlCursor, type SqlEngine, type SqlExec, type StorageRuleMetadata, type StorageRulesResult, type StudioFeaturesResult, type SubscriptionEnvelope, type SubscriptionOutcome, type SubscriptionQuery, type SystemDatabaseReader, type SystemDoc, type SystemQuery, type SystemReaderOptions, type SystemReaderSchedulerLike, type SystemReaderStorageLike, type SystemTableName, type TableColumnsResult, type TableDefinitionLike, type TableIndexInfo, type TableIndexesResult, type TableInfo, type TablePage, type TableReaderLike, type TablesColumnsResult, type TransactionSqlLike, type TriggerContextLike, type TriggerDefinitionLike, type TriggerEventLike, type TriggerOpLike, type TriggerTimingLike, type ValidatorLike, type WhereInput, type WhereSqlStrategy, type WithInput, type WorkflowMetadata, type WorkflowsResult, type WriteEvent, type WriteHook, aggregateSqlFunction, aggregateTableName, applyCdcChanges, applyOnDelete, applySelect, armRestore, assertFlatPredicate, assertReadonly, assertShapeShardable, assertValidClientId, backfillAggregateIndexes, backfillRankIndexes, buildFtsMatch, buildSecurityAudit, buildSeekWhere, clearCapturedMail, coerceAggregateNumber, compileWhereSql, containsRelationPredicate, createDependencyTracker, createShardCtxDb, createSystemReader, decodeCursor, depKey, diffExternalSource, encodeAggregateKey, encodeCursor, encodePartitionKey, ensureAuthMetricsTables, ensureFunctionMetricsTables, ensureMailTable, exportShardRows, exportShardTable, facetColumn, fanOutScalarCounts, foldAggregateTally, ftsTableName, guardWriter, hasTrigger, importShardRows, isRelationPredicate, isSoftDeleted, isSourceDue, liftSourceId, listTables, matchesRankStaticWhere, matchesStaticWhere, materializeExternalRows, materializeExternalRowsIncremental, mergeWhere, normalizeCountArgument, normalizeIdStructurally, normalizeOrderKeys, parseExportShardArgs, parseImportShardArgs, planAggregateLookup, pullExternalSourceIncrementalTick, pullExternalSourceTick, rankTableName, reactiveCacheKey, readAggregateValue, readAuthMetrics, readBookmark, readCapturedMail, readCdcChanges, readExternalSourceBaseline, readFunctionMetricBuckets, readFunctionMetricIndexHits, readFunctionMetrics, readFunctionMetricsTotals, readMigrationStatus, readTablePage, recordAuthEvent, recordCapturedMail, recordFunctionMetric, renderSql, resolveRankPartition, resolveRelationPredicates, resolveWith, runDataMigration, runExternalSourceTick, runReadonlySql, runRowValidators, runShardMigrations, runTriggers, scoreDocument, selectExportTables, selectIndexForAggregate, selectIndexForCount, selectIndexForGroupBy, selectMatchingIds, serveRelationFanout, softDeleteScope, sortColumnName, stableStringify, stableWireKey, stringifySearchText, subscriptionListDeltas, throwingScheduler, tokenizeSearch, trimCdcChanges, validateImportRow };

package/dist/index.d.ts

@@ -2407,6 +2407,11 @@ interface MaterializeResult {
   /** `id → projected-value JSON` — the post-tick membership, to feed back as `baseline` next tick. */
   nextBaseline: Map<string, string>;
 }
+/** The outcome of one incremental materialize pass. No baseline: incremental applies only the pulled slice, never a full-membership diff. */
+interface IncrementalMaterializeResult {
+  /** Number of `CdcChange`s applied (upserts + tombstone deletes) for the pulled slice. */
+  applied: number;
+}
 /**
 * Diff `pulled` against `baseline` and apply the delta to `writer`. Returns the
 * applied count and the next baseline. A steady-state tick (membership unchanged)
@@ -2417,6 +2422,36 @@ declare const materializeExternalRows: (writer: DatabaseWriterLike, pulled: Read
   table: string;
 }) => Promise<MaterializeResult>;
 /**
+* Apply an **incremental** slice (plan 136): the freshly-pulled rows changed since
+* the watermark, upsert-only. Unlike {@link materializeExternalRows} this reads no
+* baseline and never diffs the full membership — an absent row means "unchanged
+* since the watermark", NOT "deleted". Each pulled row is projected with the SAME
+* {@link projectExternalSourceRow} full-pull uses, so an incrementally-upserted row
+* is byte-identical to how the next reconcile sweep's full-pull would store it (no
+* spurious update on reconcile).
+*
+* Delete visibility comes from `deletedIds` — the ids the caller resolved from the
+* source's soft-delete tombstone column. Every other pulled row is an `insert`,
+* which {@link applyCdcChanges} upserts (insert, or replace on conflict), so a
+* changed existing row is updated and a genuinely new row is inserted without the
+* caller tracking which is which.
+*
+* **Content short-circuit.** An incremental cursor query uses `>= watermark` (so
+* rows sharing the boundary value are never skipped), which means a steady-state
+* tick re-pulls the boundary row(s) unchanged. Blindly upserting them would append
+* a `__cdc_log` entry, broadcast a spurious `update` to every `defineShape`
+* subscriber, re-run search/aggregate/rank sync, and fire `onWrite` (a Vectorize
+* re-embed = real cost) on every tick — and `replace` would reset `_creationTime`.
+* So each row is diffed against its stored projection (the SAME
+* {@link projectExternalSourceRow} + {@link stableStringify} full-pull uses) and
+* skipped when byte-identical — mirroring the full-pull diff's steady-state no-op.
+*/
+declare const materializeExternalRowsIncremental: (writer: DatabaseWriterLike, pulled: ReadonlyArray<Record<string, unknown>>, options: {
+  columns?: ReadonlyArray<string>;
+  deletedIds?: ReadonlySet<string>;
+  table: string;
+}) => Promise<IncrementalMaterializeResult>;
+/**
 * Read the materialized table's current membership as the canonical full-pull
 * baseline (`id → canonical JSON`). Reuses the shape scanner (`selectShapeRows`)
 * and the SAME {@link projectExternalSourceRow} + {@link stableStringify} the diff
@@ -2445,14 +2480,23 @@ interface SourceClientLike {
 type SourceRefresh = "manual" | {
   everyMs: number;
 };
+/** The incremental cursor config (plan 136): the watermark column + the watermark-parameterized pull query. */
+interface SourceCursorLike {
+  column: string;
+  query: string;
+}
 /** The runtime `.source(...)` config the poll loop reads — a structural mirror of `@lunora/server`'s `ExternalSourceDefinition` (only the fields the tick uses). */
 interface ExternalSourceLike {
   binding: string;
   columns?: ReadonlyArray<string>;
+  cursor?: SourceCursorLike;
   idColumn?: string;
   map?: (row: Record<string, unknown>) => Record<string, unknown>;
+  mode?: string;
   query: string;
+  reconcileEveryMs?: number;
   refresh?: SourceRefresh;
+  softDeleteColumn?: string;
   tenantBy?: (shardKey: string) => ReadonlyArray<unknown>;
 }
 /**
@@ -2482,6 +2526,39 @@ declare const isSourceDue: (refresh: SourceRefresh | undefined, lastPolledMs: nu
 */
 declare const pullExternalSourceTick: (sql: SqlExec, writer: DatabaseWriterLike, client: SourceClientLike, table: string, source: ExternalSourceLike, shardKey: string) => Promise<MaterializeResult>;
 /**
+* Whether an upstream row is a soft-delete tombstone under `column`. A set
+* `deleted_at` (any non-null value, e.g. a timestamp), an `is_deleted = true`, or a
+* non-zero flag all read as deleted; `null` / `undefined` / `false` / `0` mean
+* live. Note an **empty string** reads as deleted (`"" !== 0`), so an upstream that
+* clears the column to `""` rather than `NULL` for a live row would mis-signal —
+* use `NULL` for live rows. The incremental query MUST return tombstoned rows
+* (don't filter `WHERE deleted_at IS NULL`) or the delete is never observed.
+*/
+declare const isSoftDeleted: (row: Record<string, unknown>, column: string) => boolean;
+/**
+* Run one **incremental** tick (plan 136). Reads the durable watermark for
+* `(table, shardKey)`; on the first ever poll or when the `reconcileEveryMs` sweep
+* is due it runs a **full-pull** (seed/GC: {@link runExternalSourceTick} observes
+* deletes and re-establishes membership), otherwise it pulls only rows past the
+* watermark via `cursor.query` and upserts them ({@link materializeExternalRowsIncremental},
+* tombstones → deletes). Either way it advances the watermark to the max cursor
+* value seen and persists it (and the reconcile timestamp).
+*
+* **Crash safety** is by ordering + idempotency, NOT an atomic transaction across
+* the two write channels (the apply goes through the `writer`; the watermark write
+* is a raw `sql` write): the watermark advances only AFTER a fully-applied slice,
+* so a crash between the apply and the watermark write leaves the watermark behind
+* and the next tick re-pulls the same `>= watermark` slice — which the upsert
+* short-circuits (unchanged rows) or re-applies idempotently. It only ever replays,
+* never skips (same self-healing argument as `advanceClientWatermark`).
+*
+* Requires `source.cursor` (validated at `defineSchema` for incremental mode); a
+* missing cursor throws rather than silently degrading to a stuck watermark.
+*/
+declare const pullExternalSourceIncrementalTick: (sql: SqlExec, writer: DatabaseWriterLike, client: SourceClientLike, table: string, source: ExternalSourceLike, shardKey: string, nowMs: number) => Promise<{
+  applied: number;
+}>;
+/**
 * Reserved `functionPath` prefix for admin introspection RPCs. These travel
 * over the same `/_lunora/rpc` → shard `/rpc` path as ordinary functions, but
 * `ShardDO` intercepts them before user dispatch and serves them from the
@@ -6704,4 +6781,4 @@ interface WhereSqlStrategy {
 * `undefined` when the input imposes no constraint (empty `where`).
 */
 declare const compileWhereSql: (where: WhereInput | undefined, strategy: WhereSqlStrategy) => SQL | undefined;
-export { ADMIN_FUNCTIONS, ADMIN_FUNCTION_PREFIX, AGGREGATE_SQL_FUNCTION, AUTH_METRICS_BUCKETS_TABLE, AUTH_METRICS_BUCKET_MS, AUTH_METRICS_BUCKET_RETENTION, AUTH_METRICS_TABLE, type AdvisoriesResult, type AdvisoryFinding, type AggregateIndexDefinitionLike, type AggregateOp, type AggregateOptions, type AggregateResult, type AggregateTally, type ApplyOnDeleteOptions, type AuditEntry, type AuditLogResult, type AuthMetrics, type AuthMetricsBucket, type BroadcastDelta, CDC_LOG_TABLE, type CacheEntry, type CapturedMailRow, type CdcChange, type Clock, type ColumnMeta, type ColumnMetaLike, ConflictError, type CountArgs, CountRlsUnsupportedError, type CtxDbOptions, DATA_MIGRATION_STATE_TABLE, DEFAULT_MAX_RELATION_KEYS, type DataMigrationDocument, type DataMigrationLike, type DataMigrationTransform, type DatabaseWriterLike, type DependencyTracker, type DeployInfo, type ExportRow, type ExportShardAdminArgs, type ExportShardArgs, type ExternalSourceDiffResult, type ExternalSourceLike, FLAGS_FUNCTION_PREFIX, FUNCTION_METRICS_BUCKETS_TABLE, FUNCTION_METRICS_BUCKET_MS, FUNCTION_METRICS_BUCKET_RETENTION, FUNCTION_METRICS_INDEX_TABLE, FUNCTION_METRICS_TABLE, type FacetColumnOptions, type FacetColumnResult, type FacetValue, type FieldOperators, type FlagEvaluation, type FlagsResult, type FunctionCallStat, type FunctionMetricBucket, type FunctionMetricIndexHit, type FunctionStatsResult, type GroupByEntry, type GroupByOptions, type HibernatableWebSocket, type IdGenerator, type ImportError, type ImportShardAdminArgs, type ImportShardArgs, type ImportShardResult, type IndexDefinitionLike, type IndexRangeBuilderLike, LogBuffer, type LogEntry, type LogEventInput, type LogLevel, type LogSink, MAIL_RETENTION, MAIL_TABLE, MAX_SQL_ROWS, MIN_ADMIN_TOKEN_LENGTH, MIN_AUTH_SECRET_LENGTH, type MaskColumnMetadata, type MaskPoliciesResult, type MaterializeResult, type MigrationDirection, type MigrationRunResult, type MigrationStatus, type MigrationStatusRow, type MutationDelta, type NestedWith, NotFoundError, NotUniqueError, type OnDeleteActionLike, type OrderByInput, type OrderKey, type PaginationOptions, type PitrBookmarkResult, type PitrRestoreArgs, type PitrRestoreResult, type PitrStorage, type QueryArgs, type QueryPage, type QueueMetadata, type QueuesResult, RANK_TIEBREAK, RELATION_FUNCTION_PREFIX, RLS_UNWRAP_SYMBOL, ROOT_DO_SIZE_WARN_BYTES, ROOT_SHARD_NAME, type RankDirection, type RankIndexDefinitionLike, type RankOptions, type RankPage, type RankPageOptions, type RankPageRow, type RankPageRowKey, type RankResult, type RankSortKeyLike, ReactiveCache, type ReactiveCacheOptions, type ReadHook, type ReadTablePageOptions, type RecordAuthEventInput, type RecordFunctionMetricInput, type RecordMailInput, type RelationDefinitionLike, type RenderedSql, type ResolveRelationPredicatesOptions, type ResolveWithOptions, type RestrictableQueryOptions, type RlsPoliciesResult, type RlsPolicyMetadata, RlsRequiredError, type RlsRoleMetadata, type RpcRequest, type RunDataMigrationOptions, type RunShardApplyCdcArgs, type RunShardApplyCdcResult, type RunShardBulkDeleteArgs, type RunShardBulkDeleteResult, type RunShardExportArgs, type RunShardImportArgs, type RunShardMigrationArgs, type RunShardRankBeforeArgs, type RunShardRankPageArgs, type RunShardWriteArgs, type RunShardWriteResult, type RunTriggersOptions, SCAN_DEP, SESSION_DO_TTL_DEFAULT, SHARD_REGISTRY_DO_NAME, type SchedulableWorkflowReferenceLike, type ScheduledFunctionDoc, type SchedulerLike, type SchemaLike, type SearchFilterBuilderLike, type SecurityAuditResult, type SecurityFinding, type SecurityFindingKind, type SecurityFindingLevel, type SelectMatchingIdsOptions, type ServerDefaultContextLike, SessionDO, type SessionRecord, type SettingEntry, type SettingKind, type SettingsResult, type ShapeSubscriptionQuery, ShardDO, type ShardDOOptions, type ShardDOState, type ShardRankPageResult, ShardRegistryDO, type SocketAttachment, type SortDirection, type SourceClientLike, type SourceRefresh, type SqlConsoleResult, type SqlCursor, type SqlEngine, type SqlExec, type StorageRuleMetadata, type StorageRulesResult, type StudioFeaturesResult, type SubscriptionEnvelope, type SubscriptionOutcome, type SubscriptionQuery, type SystemDatabaseReader, type SystemDoc, type SystemQuery, type SystemReaderOptions, type SystemReaderSchedulerLike, type SystemReaderStorageLike, type SystemTableName, type TableColumnsResult, type TableDefinitionLike, type TableIndexInfo, type TableIndexesResult, type TableInfo, type TablePage, type TableReaderLike, type TablesColumnsResult, type TransactionSqlLike, type TriggerContextLike, type TriggerDefinitionLike, type TriggerEventLike, type TriggerOpLike, type TriggerTimingLike, type ValidatorLike, type WhereInput, type WhereSqlStrategy, type WithInput, type WorkflowMetadata, type WorkflowsResult, type WriteEvent, type WriteHook, aggregateSqlFunction, aggregateTableName, applyCdcChanges, applyOnDelete, applySelect, armRestore, assertFlatPredicate, assertReadonly, assertShapeShardable, assertValidClientId, backfillAggregateIndexes, backfillRankIndexes, buildFtsMatch, buildSecurityAudit, buildSeekWhere, clearCapturedMail, coerceAggregateNumber, compileWhereSql, containsRelationPredicate, createDependencyTracker, createShardCtxDb, createSystemReader, decodeCursor, depKey, diffExternalSource, encodeAggregateKey, encodeCursor, encodePartitionKey, ensureAuthMetricsTables, ensureFunctionMetricsTables, ensureMailTable, exportShardRows, exportShardTable, facetColumn, fanOutScalarCounts, foldAggregateTally, ftsTableName, guardWriter, hasTrigger, importShardRows, isRelationPredicate, isSourceDue, liftSourceId, listTables, matchesRankStaticWhere, matchesStaticWhere, materializeExternalRows, mergeWhere, normalizeCountArgument, normalizeIdStructurally, normalizeOrderKeys, parseExportShardArgs, parseImportShardArgs, planAggregateLookup, pullExternalSourceTick, rankTableName, reactiveCacheKey, readAggregateValue, readAuthMetrics, readBookmark, readCapturedMail, readCdcChanges, readExternalSourceBaseline, readFunctionMetricBuckets, readFunctionMetricIndexHits, readFunctionMetrics, readFunctionMetricsTotals, readMigrationStatus, readTablePage, recordAuthEvent, recordCapturedMail, recordFunctionMetric, renderSql, resolveRankPartition, resolveRelationPredicates, resolveWith, runDataMigration, runExternalSourceTick, runReadonlySql, runRowValidators, runShardMigrations, runTriggers, scoreDocument, selectExportTables, selectIndexForAggregate, selectIndexForCount, selectIndexForGroupBy, selectMatchingIds, serveRelationFanout, softDeleteScope, sortColumnName, stableStringify, stableWireKey, stringifySearchText, subscriptionListDeltas, throwingScheduler, tokenizeSearch, trimCdcChanges, validateImportRow };
+export { ADMIN_FUNCTIONS, ADMIN_FUNCTION_PREFIX, AGGREGATE_SQL_FUNCTION, AUTH_METRICS_BUCKETS_TABLE, AUTH_METRICS_BUCKET_MS, AUTH_METRICS_BUCKET_RETENTION, AUTH_METRICS_TABLE, type AdvisoriesResult, type AdvisoryFinding, type AggregateIndexDefinitionLike, type AggregateOp, type AggregateOptions, type AggregateResult, type AggregateTally, type ApplyOnDeleteOptions, type AuditEntry, type AuditLogResult, type AuthMetrics, type AuthMetricsBucket, type BroadcastDelta, CDC_LOG_TABLE, type CacheEntry, type CapturedMailRow, type CdcChange, type Clock, type ColumnMeta, type ColumnMetaLike, ConflictError, type CountArgs, CountRlsUnsupportedError, type CtxDbOptions, DATA_MIGRATION_STATE_TABLE, DEFAULT_MAX_RELATION_KEYS, type DataMigrationDocument, type DataMigrationLike, type DataMigrationTransform, type DatabaseWriterLike, type DependencyTracker, type DeployInfo, type ExportRow, type ExportShardAdminArgs, type ExportShardArgs, type ExternalSourceDiffResult, type ExternalSourceLike, FLAGS_FUNCTION_PREFIX, FUNCTION_METRICS_BUCKETS_TABLE, FUNCTION_METRICS_BUCKET_MS, FUNCTION_METRICS_BUCKET_RETENTION, FUNCTION_METRICS_INDEX_TABLE, FUNCTION_METRICS_TABLE, type FacetColumnOptions, type FacetColumnResult, type FacetValue, type FieldOperators, type FlagEvaluation, type FlagsResult, type FunctionCallStat, type FunctionMetricBucket, type FunctionMetricIndexHit, type FunctionStatsResult, type GroupByEntry, type GroupByOptions, type HibernatableWebSocket, type IdGenerator, type ImportError, type ImportShardAdminArgs, type ImportShardArgs, type ImportShardResult, type IncrementalMaterializeResult, type IndexDefinitionLike, type IndexRangeBuilderLike, LogBuffer, type LogEntry, type LogEventInput, type LogLevel, type LogSink, MAIL_RETENTION, MAIL_TABLE, MAX_SQL_ROWS, MIN_ADMIN_TOKEN_LENGTH, MIN_AUTH_SECRET_LENGTH, type MaskColumnMetadata, type MaskPoliciesResult, type MaterializeResult, type MigrationDirection, type MigrationRunResult, type MigrationStatus, type MigrationStatusRow, type MutationDelta, type NestedWith, NotFoundError, NotUniqueError, type OnDeleteActionLike, type OrderByInput, type OrderKey, type PaginationOptions, type PitrBookmarkResult, type PitrRestoreArgs, type PitrRestoreResult, type PitrStorage, type QueryArgs, type QueryPage, type QueueMetadata, type QueuesResult, RANK_TIEBREAK, RELATION_FUNCTION_PREFIX, RLS_UNWRAP_SYMBOL, ROOT_DO_SIZE_WARN_BYTES, ROOT_SHARD_NAME, type RankDirection, type RankIndexDefinitionLike, type RankOptions, type RankPage, type RankPageOptions, type RankPageRow, type RankPageRowKey, type RankResult, type RankSortKeyLike, ReactiveCache, type ReactiveCacheOptions, type ReadHook, type ReadTablePageOptions, type RecordAuthEventInput, type RecordFunctionMetricInput, type RecordMailInput, type RelationDefinitionLike, type RenderedSql, type ResolveRelationPredicatesOptions, type ResolveWithOptions, type RestrictableQueryOptions, type RlsPoliciesResult, type RlsPolicyMetadata, RlsRequiredError, type RlsRoleMetadata, type RpcRequest, type RunDataMigrationOptions, type RunShardApplyCdcArgs, type RunShardApplyCdcResult, type RunShardBulkDeleteArgs, type RunShardBulkDeleteResult, type RunShardExportArgs, type RunShardImportArgs, type RunShardMigrationArgs, type RunShardRankBeforeArgs, type RunShardRankPageArgs, type RunShardWriteArgs, type RunShardWriteResult, type RunTriggersOptions, SCAN_DEP, SESSION_DO_TTL_DEFAULT, SHARD_REGISTRY_DO_NAME, type SchedulableWorkflowReferenceLike, type ScheduledFunctionDoc, type SchedulerLike, type SchemaLike, type SearchFilterBuilderLike, type SecurityAuditResult, type SecurityFinding, type SecurityFindingKind, type SecurityFindingLevel, type SelectMatchingIdsOptions, type ServerDefaultContextLike, SessionDO, type SessionRecord, type SettingEntry, type SettingKind, type SettingsResult, type ShapeSubscriptionQuery, ShardDO, type ShardDOOptions, type ShardDOState, type ShardRankPageResult, ShardRegistryDO, type SocketAttachment, type SortDirection, type SourceClientLike, type SourceCursorLike, type SourceRefresh, type SqlConsoleResult, type SqlCursor, type SqlEngine, type SqlExec, type StorageRuleMetadata, type StorageRulesResult, type StudioFeaturesResult, type SubscriptionEnvelope, type SubscriptionOutcome, type SubscriptionQuery, type SystemDatabaseReader, type SystemDoc, type SystemQuery, type SystemReaderOptions, type SystemReaderSchedulerLike, type SystemReaderStorageLike, type SystemTableName, type TableColumnsResult, type TableDefinitionLike, type TableIndexInfo, type TableIndexesResult, type TableInfo, type TablePage, type TableReaderLike, type TablesColumnsResult, type TransactionSqlLike, type TriggerContextLike, type TriggerDefinitionLike, type TriggerEventLike, type TriggerOpLike, type TriggerTimingLike, type ValidatorLike, type WhereInput, type WhereSqlStrategy, type WithInput, type WorkflowMetadata, type WorkflowsResult, type WriteEvent, type WriteHook, aggregateSqlFunction, aggregateTableName, applyCdcChanges, applyOnDelete, applySelect, armRestore, assertFlatPredicate, assertReadonly, assertShapeShardable, assertValidClientId, backfillAggregateIndexes, backfillRankIndexes, buildFtsMatch, buildSecurityAudit, buildSeekWhere, clearCapturedMail, coerceAggregateNumber, compileWhereSql, containsRelationPredicate, createDependencyTracker, createShardCtxDb, createSystemReader, decodeCursor, depKey, diffExternalSource, encodeAggregateKey, encodeCursor, encodePartitionKey, ensureAuthMetricsTables, ensureFunctionMetricsTables, ensureMailTable, exportShardRows, exportShardTable, facetColumn, fanOutScalarCounts, foldAggregateTally, ftsTableName, guardWriter, hasTrigger, importShardRows, isRelationPredicate, isSoftDeleted, isSourceDue, liftSourceId, listTables, matchesRankStaticWhere, matchesStaticWhere, materializeExternalRows, materializeExternalRowsIncremental, mergeWhere, normalizeCountArgument, normalizeIdStructurally, normalizeOrderKeys, parseExportShardArgs, parseImportShardArgs, planAggregateLookup, pullExternalSourceIncrementalTick, pullExternalSourceTick, rankTableName, reactiveCacheKey, readAggregateValue, readAuthMetrics, readBookmark, readCapturedMail, readCdcChanges, readExternalSourceBaseline, readFunctionMetricBuckets, readFunctionMetricIndexHits, readFunctionMetrics, readFunctionMetricsTotals, readMigrationStatus, readTablePage, recordAuthEvent, recordCapturedMail, recordFunctionMetric, renderSql, resolveRankPartition, resolveRelationPredicates, resolveWith, runDataMigration, runExternalSourceTick, runReadonlySql, runRowValidators, runShardMigrations, runTriggers, scoreDocument, selectExportTables, selectIndexForAggregate, selectIndexForCount, selectIndexForGroupBy, selectMatchingIds, serveRelationFanout, softDeleteScope, sortColumnName, stableStringify, stableWireKey, stringifySearchText, subscriptionListDeltas, throwingScheduler, tokenizeSearch, trimCdcChanges, validateImportRow };

package/dist/index.mjs

@@ -8,8 +8,8 @@ export { DATA_MIGRATION_STATE_TABLE, readMigrationStatus, runDataMigration } fro
 export { SCAN_DEP, createDependencyTracker, depKey } from './packem_shared/SCAN_DEP-DLJF8dsj.mjs';
 export { renderSql } from './packem_shared/renderSql-D6eUcn2N.mjs';
 export { diffExternalSource } from './packem_shared/diffExternalSource-CovHfdyo.mjs';
-export { materializeExternalRows, readExternalSourceBaseline, runExternalSourceTick } from './packem_shared/materializeExternalRows-CoGmFmsY.mjs';
-export { isSourceDue, liftSourceId, pullExternalSourceTick } from './packem_shared/isSourceDue-Bj7I3v1b.mjs';
+export { materializeExternalRows, materializeExternalRowsIncremental, readExternalSourceBaseline, runExternalSourceTick } from './packem_shared/materializeExternalRows-FyA5Rwn4.mjs';
+export { isSoftDeleted, isSourceDue, liftSourceId, pullExternalSourceIncrementalTick, pullExternalSourceTick } from './packem_shared/isSoftDeleted-i5kKh6Up.mjs';
 export { FUNCTION_METRICS_BUCKETS_TABLE, FUNCTION_METRICS_BUCKET_MS, FUNCTION_METRICS_BUCKET_RETENTION, FUNCTION_METRICS_INDEX_TABLE, FUNCTION_METRICS_TABLE, ensureFunctionMetricsTables, readFunctionMetricBuckets, readFunctionMetricIndexHits, readFunctionMetrics, readFunctionMetricsTotals, recordFunctionMetric } from './packem_shared/FUNCTION_METRICS_BUCKETS_TABLE-UDNVD7FS.mjs';
 export { ADMIN_FUNCTIONS, ADMIN_FUNCTION_PREFIX, FLAGS_FUNCTION_PREFIX, RELATION_FUNCTION_PREFIX, facetColumn, listTables, readTablePage, selectMatchingIds } from './packem_shared/ADMIN_FUNCTIONS-CAHLZMj8.mjs';
 export { LogBuffer } from './packem_shared/LogBuffer-B_Ezju_N.mjs';

package/dist/packem_shared/isSoftDeleted-i5kKh6Up.mjs

@@ -0,0 +1,183 @@
+import { LunoraError } from '@lunora/errors';
+import { sql } from 'drizzle-orm';
+import { r as runDrizzle } from './do-exec-5eQy5cEi.mjs';
+import { runExternalSourceTick, materializeExternalRowsIncremental } from './materializeExternalRows-FyA5Rwn4.mjs';
+
+const SOURCE_CURSOR_TABLE = "__lunora_source_cursor";
+const serializeCursor = (value) => {
+  if (value instanceof Date) {
+    return `d:${value.toISOString()}`;
+  }
+  if (typeof value === "bigint") {
+    return `b:${value.toString()}`;
+  }
+  if (typeof value === "number") {
+    return `n:${value.toString()}`;
+  }
+  return `s:${value}`;
+};
+const deserializeCursor = (text) => {
+  const rest = text.slice(2);
+  switch (text[0]) {
+    case "b": {
+      return BigInt(rest);
+    }
+    case "d": {
+      return new Date(rest);
+    }
+    case "n": {
+      return Number(rest);
+    }
+    default: {
+      return rest;
+    }
+  }
+};
+const INTEGER_STRING = /^-?\d+$/;
+const DECIMAL_STRING = /^-?\d+(?:\.\d+)?$/;
+const cursorAfter = (a, b) => {
+  if (a instanceof Date && b instanceof Date) {
+    return a.getTime() > b.getTime();
+  }
+  if (typeof a === "bigint" && typeof b === "bigint") {
+    return a > b;
+  }
+  if (typeof a === "number" && typeof b === "number") {
+    return a > b;
+  }
+  if (typeof a === "string" && typeof b === "string" && DECIMAL_STRING.test(a) && DECIMAL_STRING.test(b)) {
+    if (INTEGER_STRING.test(a) && INTEGER_STRING.test(b)) {
+      return BigInt(a) > BigInt(b);
+    }
+    return Number(a) > Number(b);
+  }
+  return String(a) > String(b);
+};
+const maxCursorValue = (rows, column, current) => {
+  let best = current === null ? void 0 : deserializeCursor(current);
+  for (const row of rows) {
+    const raw = row[column];
+    if (raw === null || raw === void 0) {
+      continue;
+    }
+    const value = raw;
+    if (best === void 0 || cursorAfter(value, best)) {
+      best = value;
+    }
+  }
+  return best === void 0 ? null : serializeCursor(best);
+};
+const migrateSourceCursor = (sql$1) => {
+  runDrizzle(
+    sql$1,
+    sql`CREATE TABLE IF NOT EXISTS ${sql.identifier(SOURCE_CURSOR_TABLE)} (
+            table_name TEXT NOT NULL,
+            shard_key TEXT NOT NULL,
+            watermark TEXT,
+            last_reconcile_ms INTEGER,
+            PRIMARY KEY (table_name, shard_key)
+        )`
+  );
+};
+const readSourceCursor = (sql$1, table, shardKey) => {
+  const rows = runDrizzle(
+    sql$1,
+    sql`SELECT watermark, last_reconcile_ms FROM ${sql.identifier(SOURCE_CURSOR_TABLE)} WHERE table_name = ${table} AND shard_key = ${shardKey} LIMIT 1`
+  ).toArray();
+  const row = rows[0];
+  return { lastReconcileMs: row?.last_reconcile_ms ?? null, watermark: row?.watermark ?? null };
+};
+const writeSourceCursor = (sql$1, table, shardKey, state) => {
+  runDrizzle(
+    sql$1,
+    sql`INSERT INTO ${sql.identifier(SOURCE_CURSOR_TABLE)} (table_name, shard_key, watermark, last_reconcile_ms)
+             VALUES (${table}, ${shardKey}, ${state.watermark}, ${state.lastReconcileMs})
+             ON CONFLICT(table_name, shard_key) DO UPDATE SET watermark = excluded.watermark, last_reconcile_ms = excluded.last_reconcile_ms`
+  );
+};
+
+const liftSourceId = (row, options = {}) => {
+  const { idColumn = "id", map } = options;
+  const idValue = row[idColumn];
+  if (idValue === void 0 || idValue === null) {
+    throw new LunoraError("INTERNAL", `external-source: row is missing id column "${idColumn}"`);
+  }
+  if (typeof idValue !== "string" && typeof idValue !== "number" && typeof idValue !== "bigint") {
+    throw new TypeError(`external-source: id column "${idColumn}" must be a string or number`);
+  }
+  const id = String(idValue);
+  if (map) {
+    return { ...map(row), _id: id };
+  }
+  const body = {};
+  for (const [key, value] of Object.entries(row)) {
+    if (key !== idColumn) {
+      body[key] = value;
+    }
+  }
+  return { ...body, _id: id };
+};
+const isSourceDue = (refresh, lastPolledMs, nowMs) => {
+  if (refresh === "manual") {
+    return false;
+  }
+  if (refresh === void 0 || lastPolledMs === void 0) {
+    return true;
+  }
+  return nowMs - lastPolledMs >= refresh.everyMs;
+};
+const pullAndLift = async (client, query, parameters, source) => {
+  const rows = await client.query(query, parameters);
+  const documents = rows.map((row) => liftSourceId(row, { idColumn: source.idColumn, map: source.map }));
+  return { documents, rows };
+};
+const pullExternalSourceTick = async (sql, writer, client, table, source, shardKey) => {
+  const parameters = source.tenantBy ? source.tenantBy(shardKey) : [];
+  const { documents } = await pullAndLift(client, source.query, parameters, source);
+  return runExternalSourceTick(sql, writer, documents, { columns: source.columns, table });
+};
+const isSoftDeleted = (row, column) => {
+  const value = row[column];
+  return value !== null && value !== void 0 && value !== false && value !== 0;
+};
+const pullExternalSourceIncrementalTick = async (sql, writer, client, table, source, shardKey, nowMs) => {
+  const { cursor } = source;
+  if (!cursor) {
+    throw new LunoraError(
+      "INTERNAL",
+      `external-source: table "${table}" is mode "incremental" but has no \`cursor\` — this should have been rejected at defineSchema`
+    );
+  }
+  migrateSourceCursor(sql);
+  const state = readSourceCursor(sql, table, shardKey);
+  const tenantParameters = source.tenantBy ? source.tenantBy(shardKey) : [];
+  const reconcileDue = source.reconcileEveryMs !== void 0 && (state.lastReconcileMs === null || nowMs - state.lastReconcileMs >= source.reconcileEveryMs);
+  const fullPull = state.watermark === null || reconcileDue;
+  let slice;
+  let applied;
+  if (state.watermark === null || reconcileDue) {
+    slice = await pullAndLift(client, source.query, tenantParameters, source);
+    ({ applied } = await runExternalSourceTick(sql, writer, slice.documents, { columns: source.columns, table }));
+  } else {
+    slice = await pullAndLift(client, cursor.query, [...tenantParameters, deserializeCursor(state.watermark)], source);
+    const { softDeleteColumn } = source;
+    const deletedIds = softDeleteColumn ? new Set(
+      slice.documents.flatMap((document, index) => {
+        const row = slice.rows[index];
+        return row && isSoftDeleted(row, softDeleteColumn) ? [String(document._id)] : [];
+      })
+    ) : void 0;
+    ({ applied } = await materializeExternalRowsIncremental(writer, slice.documents, { columns: source.columns, deletedIds, table }));
+  }
+  const watermark = maxCursorValue(slice.rows, cursor.column, state.watermark);
+  if (fullPull && watermark === null && slice.rows.length > 0) {
+    throw new LunoraError(
+      "INTERNAL",
+      `external-source: table "${table}" (mode "incremental") pulled ${String(slice.rows.length)} rows but none carry the cursor column "${cursor.column}" — the seed \`query\` must project it (matching \`cursor.query\`'s alias), or the watermark can never advance.`
+    );
+  }
+  writeSourceCursor(sql, table, shardKey, { lastReconcileMs: fullPull ? nowMs : state.lastReconcileMs, watermark });
+  return { applied };
+};
+
+export { isSoftDeleted, isSourceDue, liftSourceId, pullExternalSourceIncrementalTick, pullExternalSourceTick };

package/dist/packem_shared/{materializeExternalRows-CoGmFmsY.mjs → materializeExternalRows-FyA5Rwn4.mjs}

@@ -8,6 +8,25 @@ const materializeExternalRows = async (writer, pulled, baseline, options) => {
   await applyCdcChanges(writer, changes);
   return { applied: changes.length, nextBaseline };
 };
+const materializeExternalRowsIncremental = async (writer, pulled, options) => {
+  const { columns, deletedIds, table } = options;
+  const changes = [];
+  for (const source of pulled) {
+    const value = projectExternalSourceRow(source, columns);
+    const id = String(value._id);
+    if (deletedIds?.has(id)) {
+      changes.push({ id, op: "delete", seq: 0, table, ts: 0 });
+      continue;
+    }
+    const stored = await writer.get(id, table);
+    if (stored && stableStringify(projectExternalSourceRow({ ...stored, _id: id }, columns)) === stableStringify(value)) {
+      continue;
+    }
+    changes.push({ doc: value, id, op: "insert", seq: 0, table, ts: 0 });
+  }
+  await applyCdcChanges(writer, changes);
+  return { applied: changes.length };
+};
 const readExternalSourceBaseline = (sql, table, columns) => {
   const baseline = /* @__PURE__ */ new Map();
   for (const { doc, id } of selectShapeRows(sql, table, void 0)) {
@@ -20,4 +39,4 @@ const runExternalSourceTick = async (sql, writer, pulled, options) => {
   return materializeExternalRows(writer, pulled, baseline, options);
 };
 
-export { materializeExternalRows, readExternalSourceBaseline, runExternalSourceTick };
+export { materializeExternalRows, materializeExternalRowsIncremental, readExternalSourceBaseline, runExternalSourceTick };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lunora/do",
-  "version": "1.0.0-alpha.31",
+  "version": "1.0.0-alpha.32",
   "description": "Lunora Durable Objects: ShardDO (SQLite, OCC, hibernated WebSocket subscriptions) and SessionDO",
   "keywords": [
     "cloudflare",

package/dist/packem_shared/isSourceDue-Bj7I3v1b.mjs

@@ -1,41 +0,0 @@
-import { LunoraError } from '@lunora/errors';
-import { runExternalSourceTick } from './materializeExternalRows-CoGmFmsY.mjs';
-
-const liftSourceId = (row, options = {}) => {
-  const { idColumn = "id", map } = options;
-  const idValue = row[idColumn];
-  if (idValue === void 0 || idValue === null) {
-    throw new LunoraError("INTERNAL", `external-source: row is missing id column "${idColumn}"`);
-  }
-  if (typeof idValue !== "string" && typeof idValue !== "number" && typeof idValue !== "bigint") {
-    throw new TypeError(`external-source: id column "${idColumn}" must be a string or number`);
-  }
-  const id = String(idValue);
-  if (map) {
-    return { ...map(row), _id: id };
-  }
-  const body = {};
-  for (const [key, value] of Object.entries(row)) {
-    if (key !== idColumn) {
-      body[key] = value;
-    }
-  }
-  return { ...body, _id: id };
-};
-const isSourceDue = (refresh, lastPolledMs, nowMs) => {
-  if (refresh === "manual") {
-    return false;
-  }
-  if (refresh === void 0 || lastPolledMs === void 0) {
-    return true;
-  }
-  return nowMs - lastPolledMs >= refresh.everyMs;
-};
-const pullExternalSourceTick = async (sql, writer, client, table, source, shardKey) => {
-  const parameters = source.tenantBy ? source.tenantBy(shardKey) : [];
-  const rows = await client.query(source.query, parameters);
-  const documents = rows.map((row) => liftSourceId(row, { idColumn: source.idColumn, map: source.map }));
-  return runExternalSourceTick(sql, writer, documents, { columns: source.columns, table });
-};
-
-export { isSourceDue, liftSourceId, pullExternalSourceTick };