@kubun/store-connector 0.17.1 → 0.18.0

package/lib/sweep-state.js

@@ -0,0 +1,180 @@
+import { decodeJSONColumn, encodeJSONColumn } from './serialization.js';
+export const MAX_SWEEP_APPLY_ATTEMPTS = 5;
+export function createSweepStateMethods({ db, deps }) {
+    const now = deps.now ?? Date.now;
+    return {
+        async recordRelistReceipt (receipt, candidates) {
+            if (!receipt.complete) return;
+            await db.transaction().execute(async (tx)=>{
+                const existing = await tx.selectFrom('kubun_connector_sweep_receipt').select('committed_at').where('owner_did', '=', receipt.ownerDID).where('connector_name', '=', receipt.connectorName).where('partition', '=', receipt.partition).where('provenance', '=', receipt.provenance).executeTakeFirst();
+                // A replay of a committed receipt has nothing left to verify.
+                if (existing?.committed_at != null) return;
+                if (existing == null) {
+                    // Recording order per partition, so an older receipt that commits
+                    // late never moves the sweep checkpoint back.
+                    const latest = await tx.selectFrom('kubun_connector_sweep_receipt').select((eb)=>eb.fn.max('sequence').as('sequence')).where('owner_did', '=', receipt.ownerDID).where('connector_name', '=', receipt.connectorName).where('partition', '=', receipt.partition).executeTakeFirst();
+                    await tx.insertInto('kubun_connector_sweep_receipt').values({
+                        owner_did: receipt.ownerDID,
+                        connector_name: receipt.connectorName,
+                        partition: receipt.partition,
+                        provenance: receipt.provenance,
+                        sequence: Number(latest?.sequence ?? 0) + 1,
+                        model_id: receipt.modelID,
+                        scope: encodeJSONColumn(receipt.scope),
+                        keep: encodeJSONColumn(receipt.keep),
+                        source_fetch_hlc: receipt.sourceFetchHLC,
+                        predicate: encodeJSONColumn(receipt.predicate),
+                        kind: receipt.kind,
+                        complete: 1,
+                        created_at: now(),
+                        committed_at: null
+                    }).execute();
+                }
+                for (const candidate of candidates){
+                    await tx.insertInto('kubun_connector_sweep_candidate').values({
+                        owner_did: receipt.ownerDID,
+                        connector_name: receipt.connectorName,
+                        partition: receipt.partition,
+                        provenance: receipt.provenance,
+                        document_id: candidate.documentID,
+                        source_id: candidate.sourceID,
+                        model_id: candidate.modelID,
+                        state: 'pending',
+                        attempts: 0,
+                        apply_failures: 0,
+                        last_error: null,
+                        updated_at: now(),
+                        next_attempt_at: 0
+                    }).onConflict((oc)=>oc.doNothing()).execute();
+                }
+            });
+        },
+        async listSweepReceipts (key) {
+            const rows = await db.selectFrom('kubun_connector_sweep_receipt').selectAll().where('owner_did', '=', key.ownerDID).where('connector_name', '=', key.connectorName).where('committed_at', 'is', null).orderBy('created_at').execute();
+            return rows.map((row)=>({
+                    ownerDID: row.owner_did,
+                    connectorName: row.connector_name,
+                    partition: row.partition,
+                    provenance: row.provenance,
+                    modelID: row.model_id,
+                    scope: decodeJSONColumn(row.scope),
+                    keep: decodeJSONColumn(row.keep),
+                    sourceFetchHLC: row.source_fetch_hlc,
+                    predicate: decodeJSONColumn(row.predicate),
+                    complete: row.complete === 1,
+                    kind: row.kind
+                }));
+        },
+        async listSweepCandidates (key) {
+            let query = db.selectFrom('kubun_connector_sweep_candidate').selectAll().where('owner_did', '=', key.ownerDID).where('connector_name', '=', key.connectorName).orderBy('updated_at').orderBy('document_id');
+            if (key.dueAt != null) {
+                query = query.where('state', 'not in', [
+                    'applied',
+                    'refused'
+                ]).where('next_attempt_at', '<=', key.dueAt);
+            }
+            const rows = await query.execute();
+            return rows.map((row)=>({
+                    ownerDID: row.owner_did,
+                    connectorName: row.connector_name,
+                    partition: row.partition,
+                    provenance: row.provenance,
+                    documentID: row.document_id,
+                    sourceID: row.source_id,
+                    modelID: row.model_id,
+                    state: row.state,
+                    attempts: row.attempts,
+                    applyFailures: row.apply_failures,
+                    lastError: row.last_error,
+                    updatedAt: Number(row.updated_at),
+                    nextAttemptAt: Number(row.next_attempt_at)
+                }));
+        },
+        async recordSweepOutcome (input) {
+            await db.updateTable('kubun_connector_sweep_candidate').set((eb)=>({
+                    state: input.outcome,
+                    attempts: eb('attempts', '+', 1),
+                    last_error: input.lastError ?? null,
+                    updated_at: now(),
+                    next_attempt_at: input.nextAttemptAt ?? 0
+                })).where('owner_did', '=', input.ownerDID).where('connector_name', '=', input.connectorName).where('partition', '=', input.partition).where('provenance', '=', input.provenance).where('document_id', '=', input.documentID).where('state', 'not in', [
+                'applied',
+                'refused'
+            ]).execute();
+        },
+        async recordSweepApplyFailure (input) {
+            const row = await db.updateTable('kubun_connector_sweep_candidate').set((eb)=>({
+                    state: eb.case().when('apply_failures', '>=', MAX_SWEEP_APPLY_ATTEMPTS - 1).then('refused').else('pending').end(),
+                    apply_failures: eb('apply_failures', '+', 1),
+                    last_error: input.lastError,
+                    next_attempt_at: input.nextAttemptAt,
+                    updated_at: now()
+                })).where('owner_did', '=', input.ownerDID).where('connector_name', '=', input.connectorName).where('partition', '=', input.partition).where('provenance', '=', input.provenance).where('document_id', '=', input.documentID).where('state', 'not in', [
+                'applied',
+                'refused'
+            ]).returning('state').executeTakeFirst();
+            return row?.state === 'refused';
+        },
+        async markSweepCandidateApplied (key) {
+            await db.updateTable('kubun_connector_sweep_candidate').set({
+                state: 'applied',
+                last_error: null,
+                next_attempt_at: 0,
+                updated_at: now()
+            }).where('owner_did', '=', key.ownerDID).where('connector_name', '=', key.connectorName).where('partition', '=', key.partition).where('provenance', '=', key.provenance).where('document_id', '=', key.documentID).where('state', 'in', [
+                'present',
+                'absent',
+                'cancelled'
+            ]).execute();
+        },
+        async commitSweepCheckpoint (key) {
+            return db.transaction().execute(async (tx)=>{
+                const receipt = await tx.selectFrom('kubun_connector_sweep_receipt').select([
+                    'complete',
+                    'committed_at',
+                    'sequence'
+                ]).where('owner_did', '=', key.ownerDID).where('connector_name', '=', key.connectorName).where('partition', '=', key.partition).where('provenance', '=', key.provenance).executeTakeFirst();
+                if (receipt?.complete !== 1 || receipt.committed_at != null) return false;
+                const unresolved = await tx.selectFrom('kubun_connector_sweep_candidate').select('document_id').where('owner_did', '=', key.ownerDID).where('connector_name', '=', key.connectorName).where('partition', '=', key.partition).where('provenance', '=', key.provenance).where('state', 'not in', [
+                    'applied',
+                    'refused'
+                ]).executeTakeFirst();
+                if (unresolved != null) return false;
+                const sequence = Number(receipt.sequence);
+                // A later relist must not advance past an earlier unresolved candidate.
+                const earlier = await tx.selectFrom('kubun_connector_sweep_candidate as candidate').innerJoin('kubun_connector_sweep_receipt as receipt', (join)=>join.onRef('receipt.owner_did', '=', 'candidate.owner_did').onRef('receipt.connector_name', '=', 'candidate.connector_name').onRef('receipt.partition', '=', 'candidate.partition').onRef('receipt.provenance', '=', 'candidate.provenance')).select('candidate.document_id').where('candidate.owner_did', '=', key.ownerDID).where('candidate.connector_name', '=', key.connectorName).where('candidate.partition', '=', key.partition).where('candidate.state', 'not in', [
+                    'applied',
+                    'refused'
+                ]).where('receipt.sequence', '<', sequence).executeTakeFirst();
+                if (earlier != null) return false;
+                await tx.insertInto('kubun_connector_sweep_checkpoint').values({
+                    owner_did: key.ownerDID,
+                    connector_name: key.connectorName,
+                    partition: key.partition,
+                    provenance: key.provenance,
+                    receipt_sequence: sequence,
+                    updated_at: now()
+                }).onConflict((oc)=>oc.columns([
+                        'owner_did',
+                        'connector_name',
+                        'partition'
+                    ]).doUpdateSet({
+                        provenance: key.provenance,
+                        receipt_sequence: sequence,
+                        updated_at: now()
+                    }).where('kubun_connector_sweep_checkpoint.receipt_sequence', '<', sequence)).execute();
+                // Applied rows can be discarded; refused rows retain their error for inspection.
+                // The committed receipt prevents replay from re-verifying either outcome.
+                await tx.deleteFrom('kubun_connector_sweep_candidate').where('owner_did', '=', key.ownerDID).where('connector_name', '=', key.connectorName).where('partition', '=', key.partition).where('provenance', '=', key.provenance).where('state', '=', 'applied').execute();
+                await tx.updateTable('kubun_connector_sweep_receipt').set({
+                    committed_at: now()
+                }).where('owner_did', '=', key.ownerDID).where('connector_name', '=', key.connectorName).where('partition', '=', key.partition).where('provenance', '=', key.provenance).execute();
+                return true;
+            });
+        },
+        async getSweepCheckpoint (key) {
+            const row = await db.selectFrom('kubun_connector_sweep_checkpoint').select('provenance').where('owner_did', '=', key.ownerDID).where('connector_name', '=', key.connectorName).where('partition', '=', key.partition).executeTakeFirst();
+            return row?.provenance ?? null;
+        }
+    };
+}

package/lib/sync-state.d.ts

@@ -0,0 +1,4 @@
+import type { ConnectorStoreAPI } from './api-types.js';
+import type { ConnectorStoreContext } from './store-context.js';
+export type SyncStateMethods = Pick<ConnectorStoreAPI, 'getSyncState' | 'setSyncState' | 'updateSyncProgress' | 'deleteSyncState' | 'listSyncStatesForConnector'>;
+export declare function createSyncStateMethods(context: ConnectorStoreContext): SyncStateMethods;

package/lib/sync-state.js

@@ -0,0 +1,107 @@
+import { decodeJSONColumn, encodeJSONColumn, rowToSyncState } from './serialization.js';
+export function createSyncStateMethods(context) {
+    const { db, getRandomID } = context;
+    const methods = {
+        // --- Sync state operations ---
+        async getSyncState (connectorName, ownerDID) {
+            const row = await db.selectFrom('kubun_connector_sync_state').selectAll().where('connector_name', '=', connectorName).where('owner_did', '=', ownerDID).executeTakeFirst();
+            if (row == null) return null;
+            return rowToSyncState(row);
+        },
+        async setSyncState (connectorName, ownerDID, state) {
+            // `lease_expires_at` is dead: a fresh INSERT sets it null and a conflicting
+            // update leaves it untouched. A fresh INSERT seeds version 0 and a random
+            // incarnation; a conflicting update leaves both untouched (this path is not
+            // the CAS write — it does not bump version or churn the incarnation).
+            const checkpoint = encodeJSONColumn(state.checkpoint);
+            const recovery = encodeJSONColumn(state.recovery);
+            await db.insertInto('kubun_connector_sync_state').values({
+                connector_name: connectorName,
+                owner_did: ownerDID,
+                checkpoint,
+                last_synced_at: state.lastSyncedAt,
+                entity_count: state.entityCount,
+                status: state.status,
+                error: state.error ?? null,
+                lease_expires_at: null,
+                recovery,
+                version: 0,
+                incarnation: getRandomID()
+            }).onConflict((oc)=>oc.columns([
+                    'connector_name',
+                    'owner_did'
+                ]).doUpdateSet({
+                    checkpoint,
+                    last_synced_at: state.lastSyncedAt,
+                    entity_count: state.entityCount,
+                    status: state.status,
+                    error: state.error ?? null,
+                    recovery
+                })).execute();
+        },
+        async updateSyncProgress ({ connectorName, ownerDID, recompute, options }) {
+            // The caller passes the authoritative MAX_CAS_RETRIES; 5 is a defensive
+            // fallback for direct callers/tests.
+            const maxRetries = options?.maxRetries ?? 5;
+            for(let attempt = 0; attempt <= maxRetries; attempt++){
+                const row = await db.selectFrom('kubun_connector_sync_state').selectAll().where('connector_name', '=', connectorName).where('owner_did', '=', ownerDID).executeTakeFirst();
+                if (row == null) {
+                    // No row yet: INSERT ... ON CONFLICT DO NOTHING so a lost insert race
+                    // (a concurrent writer created the row first) falls through to reload
+                    // and take the update path.
+                    const result = await recompute(null);
+                    const inserted = await db.insertInto('kubun_connector_sync_state').values({
+                        connector_name: connectorName,
+                        owner_did: ownerDID,
+                        checkpoint: encodeJSONColumn(result.checkpoint),
+                        last_synced_at: result.lastSyncedAt,
+                        entity_count: result.entityCount,
+                        status: 'idle',
+                        error: result.error ?? null,
+                        lease_expires_at: null,
+                        recovery: encodeJSONColumn(result.recovery),
+                        version: 0,
+                        incarnation: getRandomID()
+                    }).onConflict((oc)=>oc.columns([
+                            'connector_name',
+                            'owner_did'
+                        ]).doNothing()).executeTakeFirst();
+                    if ((inserted?.numInsertedOrUpdatedRows ?? 0n) > 0n) return;
+                    continue;
+                }
+                const result = await recompute({
+                    checkpoint: decodeJSONColumn(row.checkpoint),
+                    recovery: decodeJSONColumn(row.recovery),
+                    version: row.version,
+                    incarnation: row.incarnation ?? ''
+                });
+                // Single conditional UPDATE guarded on the read version AND incarnation.
+                // `status` is deliberately omitted (workflow owns it). Zero rows means a
+                // concurrent writer won, or the row was delete+recreated (incarnation
+                // differs) ⇒ reload and recompute.
+                const updated = await db.updateTable('kubun_connector_sync_state').set({
+                    checkpoint: encodeJSONColumn(result.checkpoint),
+                    last_synced_at: result.lastSyncedAt,
+                    entity_count: result.entityCount,
+                    error: result.error ?? null,
+                    recovery: encodeJSONColumn(result.recovery),
+                    version: row.version + 1
+                }).where('connector_name', '=', connectorName).where('owner_did', '=', ownerDID).where('version', '=', row.version).where('incarnation', '=', row.incarnation).executeTakeFirst();
+                if ((updated?.numUpdatedRows ?? 0n) > 0n) return;
+            }
+            throw new Error(`updateSyncProgress: CAS failed after ${maxRetries + 1} attempts for ${connectorName}/${ownerDID}`);
+        },
+        async deleteSyncState (connectorName, ownerDID) {
+            await db.deleteFrom('kubun_connector_sync_state').where('connector_name', '=', connectorName).where('owner_did', '=', ownerDID).execute();
+        },
+        async listSyncStatesForConnector (connectorName) {
+            const rows = await db.selectFrom('kubun_connector_sync_state').selectAll().where('connector_name', '=', connectorName).execute();
+            return rows.map((row)=>({
+                    ...rowToSyncState(row),
+                    connectorName: row.connector_name,
+                    ownerDID: row.owner_did
+                }));
+        }
+    };
+    return methods;
+}

package/lib/tables.d.ts

@@ -13,6 +13,45 @@ export type ConnectorSyncStateTable = {
     incarnation: string | null;
 };
 export type ConnectorSyncState = Selectable<ConnectorSyncStateTable>;
+export type ConnectorSweepReceiptTable = {
+    owner_did: string;
+    connector_name: string;
+    partition: string;
+    provenance: string;
+    sequence: number;
+    model_id: string;
+    scope: string;
+    keep: string;
+    source_fetch_hlc: string;
+    predicate: string | null;
+    kind: string;
+    complete: number;
+    created_at: number;
+    committed_at: number | null;
+};
+export type ConnectorSweepCandidateTable = {
+    owner_did: string;
+    connector_name: string;
+    partition: string;
+    provenance: string;
+    document_id: string;
+    source_id: string;
+    model_id: string;
+    state: string;
+    attempts: number;
+    apply_failures: number;
+    last_error: string | null;
+    updated_at: number;
+    next_attempt_at: number;
+};
+export type ConnectorSweepCheckpointTable = {
+    owner_did: string;
+    connector_name: string;
+    partition: string;
+    provenance: string;
+    receipt_sequence: number;
+    updated_at: number;
+};
 export type ConnectorPendingAuthTable = {
     state: string;
     code_verifier: string;
@@ -34,10 +73,98 @@ export type ConnectorPendingAuthTable = {
     redirect_url: string;
     /** JSON-encoded `Array<string>`. See `PendingAuthRecord.scopes`. */
     scopes: string;
+    /** Null means the request predates incremental-mode tracking. */
+    is_incremental: number;
     expires_at: number;
 };
 export type ConnectorPendingAuth = Selectable<ConnectorPendingAuthTable>;
+export type ConnectorOperationTable = {
+    caller_did: string;
+    operation_id: string;
+    node_key: string;
+    owner_did: string;
+    connector_name: string;
+    model: string;
+    op: string;
+    account_key: string;
+    target: string;
+    source_id: string | null;
+    deterministic_event_id: string;
+    payload_hash: string;
+    reissue_intent: string;
+    provider_etag: string | null;
+    document_id: string | null;
+    result_entity: string | null;
+    state: string;
+    attempts: number;
+    next_attempt_at: number | null;
+    last_kind: string | null;
+    last_http_status: number | null;
+    defer_reason: string | null;
+    send_marker: string | null;
+    version: number;
+    incarnation: string | null;
+    lease_token: string | null;
+    lease_expires_at: number | null;
+    created_at: number;
+    updated_at: number;
+};
+export type ConnectorOperation = Selectable<ConnectorOperationTable>;
+export type ConnectorProjectionWatchTable = {
+    owner_did: string;
+    connector_name: string;
+    model: string;
+    account_key: string;
+    source_id: string;
+    document_id: string;
+    verified_state_digest: string | null;
+    observed_winner_digest: string | null;
+    verification_status: string;
+    verification_attempts: number;
+    repair_generation: number;
+    next_attempt_at: number | null;
+    claimed_until: number | null;
+    defer_reason: string | null;
+    version: number;
+    created_at: number;
+    updated_at: number;
+};
+export type ConnectorProjectionWatch = Selectable<ConnectorProjectionWatchTable>;
+export type ConnectorEntityTrimTable = {
+    document_id: string;
+    owner_did: string;
+    connector_name: string;
+    trimmed_fields: string;
+    updated_at: number;
+};
+export type ConnectorEntityTrim = Selectable<ConnectorEntityTrimTable>;
+export type ConnectorObservationRecordTable = {
+    type: string;
+    owner: string;
+    id: string;
+    op_hash: string;
+    hlc: string;
+    op_jwt: string;
+    payload: Uint8Array | null;
+};
+export type ConnectorObservationRepairTable = {
+    document_id: string;
+    owner: string;
+    fetch_hlc: string;
+    decision_hlc: string;
+    present: number;
+    value_digest: string | null;
+    written_at: number;
+};
 export type ConnectorTables = {
+    kubun_connector_sweep_receipt: ConnectorSweepReceiptTable;
+    kubun_connector_sweep_candidate: ConnectorSweepCandidateTable;
+    kubun_connector_sweep_checkpoint: ConnectorSweepCheckpointTable;
+    kubun_connector_observation_records: ConnectorObservationRecordTable;
+    kubun_connector_observation_repair: ConnectorObservationRepairTable;
     kubun_connector_sync_state: ConnectorSyncStateTable;
     kubun_connector_pending_auth: ConnectorPendingAuthTable;
+    kubun_connector_operation: ConnectorOperationTable;
+    kubun_connector_projection_watch: ConnectorProjectionWatchTable;
+    kubun_connector_entity_trim: ConnectorEntityTrimTable;
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@kubun/store-connector",
-  "version": "0.17.1",
+  "version": "0.18.0",
   "keywords": [],
   "license": "see LICENSE.md",
   "sideEffects": false,
@@ -15,14 +15,20 @@
     "LICENSE.md"
   ],
   "dependencies": {
-    "@kubun/db": "^0.17.0",
-    "@kubun/db-adapter": "^0.17.0",
-    "kysely": "^0.29.5"
+    "@kokuin/token": "^0.5.1",
+    "@kubun/db": "^0.18.0",
+    "@kubun/db-adapter": "^0.18.0",
+    "@kubun/records": "^0.18.0",
+    "kysely": "^0.29.7"
   },
   "devDependencies": {
-    "@kubun/db-better-sqlite": "^0.17.0",
-    "@kubun/db-postgres": "^0.17.0",
-    "@testcontainers/postgresql": "^12.1.0"
+    "@kubun/db-better-sqlite": "^0.18.0",
+    "@kubun/db-postgres": "^0.18.0",
+    "@kubun/id": "^0.18.0",
+    "@kubun/mutation": "^0.18.0",
+    "@kubun/protocol": "^0.18.0",
+    "@kubun/store-graph": "^0.18.0",
+    "@testcontainers/postgresql": "^12.2.1"
   },
   "publishConfig": {
     "access": "public"