@kubun/store-connector 0.17.0 → 0.17.2

package/lib/definition.d.ts

@@ -1,4 +1,9 @@
 import type { StoreDefinition } from '@kubun/db';
-import type { ConnectorStoreAPI } from './api.js';
+import type { ConnectorStoreAPI, ConnectorStoreDeps } from './api.js';
 import type { ConnectorTables } from './tables.js';
-export declare const connectorStoreDefinition: StoreDefinition<ConnectorTables, ConnectorStoreAPI>;
+/**
+ * The connector store's definition, its API bound to `deps`:
+ * `KubunDB.register`/`getStore` call `createAPI(db, adapter)` with exactly two
+ * arguments, so the dependencies close over here.
+ */
+export declare function createConnectorStoreDefinition(deps: ConnectorStoreDeps): StoreDefinition<ConnectorTables, ConnectorStoreAPI>;

package/lib/definition.js

@@ -1,7 +1,13 @@
 import { createConnectorStore } from './api.js';
 import { getConnectorMigrations } from './migrations.js';
-export const connectorStoreDefinition = {
-    name: 'connector',
-    migrations: getConnectorMigrations,
-    createAPI: createConnectorStore
-};
+/**
+ * The connector store's definition, its API bound to `deps`:
+ * `KubunDB.register`/`getStore` call `createAPI(db, adapter)` with exactly two
+ * arguments, so the dependencies close over here.
+ */ export function createConnectorStoreDefinition(deps) {
+    return {
+        name: 'connector',
+        migrations: getConnectorMigrations,
+        createAPI: (db, adapter)=>createConnectorStore(db, adapter, deps)
+    };
+}

package/lib/index.d.ts

@@ -1,8 +1,8 @@
 import type { StoreProvider } from '@kubun/db';
 import type { ConnectorStoreAPI } from './api.js';
-export type { ConnectorStoreAPI, PendingAuthRecord, SyncState, SyncStateEntry, SyncStatus, } from './api.js';
+export type { ArmWatchOptions, ArmWatchOutcome, ArmWatchReason, ClaimOperationInput, ConnectorOperation, ConnectorOperationOp, ConnectorOperationState, ConnectorStoreAPI, ConnectorStoreDeps, DeferReason, FinalizeOperationInput, PendingAuthRecord, ProjectionWatch, ProjectionWatchIdentity, ProjectionWatchVerificationStatus, ReadWinner, RecordOperationInput, RecordVerificationInput, RecordVerificationOutcome, SyncProgressCASCurrent, SyncProgressCASResult, SyncProgressRecompute, SyncState, SyncStateEntry, SyncStatus, TrackedOperationFilter, } from './api.js';
 export { createConnectorStore } from './api.js';
-export { connectorStoreDefinition } from './definition.js';
+export { createConnectorStoreDefinition } from './definition.js';
 export declare const CONNECTOR_STORE: "connector";
 export declare function getConnectorStore(provider: StoreProvider): Promise<ConnectorStoreAPI>;
 export type { ConnectorSyncState, ConnectorSyncStateTable, ConnectorTables, } from './tables.js';

package/lib/index.js

@@ -1,5 +1,5 @@
 export { createConnectorStore } from './api.js';
-export { connectorStoreDefinition } from './definition.js';
+export { createConnectorStoreDefinition } from './definition.js';
 export const CONNECTOR_STORE = 'connector';
 export function getConnectorStore(provider) {
     return provider.getStore(CONNECTOR_STORE);

package/lib/migrations.js

@@ -7,7 +7,13 @@ export function getConnectorMigrations(ctx) {
             // overflows int32 on Postgres; bigint holds it and the adapter's int8
             // parser reads it back as a JS number. Nullable: a row with no active
             // lease leaves this null.
-            .addColumn('lease_expires_at', t.bigint).addPrimaryKeyConstraint('kubun_connector_sync_state_pkey', [
+            .addColumn('lease_expires_at', t.bigint)// Durable recovery buffer, JSON-encoded (see @kubun/connector `Recovery`).
+            // Nullable: null means an empty buffer.
+            .addColumn('recovery', t.text)// Optimistic-concurrency version, bumped on every atomic checkpoint+
+            // recovery write. Starts at 0.
+            .addColumn('version', 'integer', (col)=>col.notNull().defaultTo(0))// Random id set on insert, part of the CAS predicate so a paused writer
+            // cannot rebase stale recovery onto a delete+recreated row (ABA).
+            .addColumn('incarnation', t.text).addPrimaryKeyConstraint('kubun_connector_sync_state_pkey', [
                 'connector_name',
                 'owner_did'
             ]).execute();
@@ -27,8 +33,76 @@ export function getConnectorMigrations(ctx) {
             // the adapter's int8 parser reads it back as a JS number. The sweep in
             // `deleteExpiredPendingAuth` keys off this column, not `created_at`.
             .addColumn('expires_at', t.bigint, (col)=>col.notNull()).execute();
+            // Connector write-back operation envelope: one row per attempted create/
+            // update, keyed by the caller's idempotency key so a replayed request
+            // (crash recovery, retry) converges on the same row instead of double-
+            // writing to the provider.
+            await db.schema.createTable('kubun_connector_operation').ifNotExists().addColumn('caller_did', t.text, (col)=>col.notNull()).addColumn('operation_id', t.text, (col)=>col.notNull())// An opaque random key assigned at claim time, the operation's public
+            // identity (its GraphQL node id). Kept apart from the caller-supplied
+            // `operation_id`, which is an idempotency key and says nothing about
+            // who may look the operation up.
+            .addColumn('node_key', t.text, (col)=>col.notNull()).addColumn('owner_did', t.text, (col)=>col.notNull()).addColumn('connector_name', t.text, (col)=>col.notNull()).addColumn('model', t.text, (col)=>col.notNull()).addColumn('op', t.text, (col)=>col.notNull()).addColumn('account_key', t.text, (col)=>col.notNull()).addColumn('target', t.text, (col)=>col.notNull()).addColumn('source_id', t.text).addColumn('deterministic_event_id', t.text, (col)=>col.notNull()).addColumn('payload_hash', t.text, (col)=>col.notNull()).addColumn('reissue_intent', t.text, (col)=>col.notNull()).addColumn('provider_etag', t.text).addColumn('document_id', t.text).addColumn('result_entity', t.text).addColumn('state', t.text, (col)=>col.notNull()).addColumn('attempts', 'integer', (col)=>col.notNull().defaultTo(0))// ms-epoch (Date.now) overflows int32 on Postgres; bigint holds it and
+            // the adapter's int8 parser reads it back as a JS number.
+            .addColumn('next_attempt_at', t.bigint)// Why `next_attempt_at` was pushed out without spending an attempt, when
+            // that reason can lapse on its own: 'offline' is made due again as soon
+            // as the device reconnects, 'auth' as soon as the owner authorizes.
+            // Null for a backoff or any other deferral.
+            .addColumn('defer_reason', t.text)// The run that started a provider write in the current attempt (a
+            // per-run marker), cleared when a new attempt begins: a re-run of the
+            // same attempt cannot claim no request was ever sent.
+            .addColumn('send_marker', t.text).addColumn('last_kind', t.text).addColumn('last_http_status', 'integer').addColumn('version', 'integer', (col)=>col.notNull().defaultTo(0)).addColumn('incarnation', t.text).addColumn('lease_token', t.text)// ms-epoch (Date.now) overflows int32 on Postgres; bigint holds it and
+            // the adapter's int8 parser reads it back as a JS number.
+            .addColumn('lease_expires_at', t.bigint)// ms-epoch (Date.now) overflows int32 on Postgres; bigint holds it and
+            // the adapter's int8 parser reads it back as a JS number.
+            .addColumn('created_at', t.bigint, (col)=>col.notNull()).addColumn('updated_at', t.bigint, (col)=>col.notNull()).addPrimaryKeyConstraint('kubun_connector_operation_pkey', [
+                'caller_did',
+                'operation_id'
+            ]).execute();
+            await db.schema.createIndex('uq_connector_operation_node_key').ifNotExists().on('kubun_connector_operation').column('node_key').unique().execute();
+            // The operations of a document (its write state), read by document id.
+            await db.schema.createIndex('idx_connector_operation_document').ifNotExists().on('kubun_connector_operation').column('document_id').execute();
+            // The operations an owner may see (`listTrackedOperations({ visibleTo })`
+            // matches the caller through the primary key, the owner through this).
+            await db.schema.createIndex('idx_connector_operation_owner').ifNotExists().on('kubun_connector_operation').column('owner_did').execute();
+            // The retention cleanup's candidates: finished operations by last update.
+            await db.schema.createIndex('idx_connector_operation_state_updated').ifNotExists().on('kubun_connector_operation').columns([
+                'state',
+                'updated_at'
+            ]).execute();
+            // Connector projection verify/repair watch: one row per ingested
+            // provider document being tracked for eventual verification against the
+            // local converged state, with a unique ingress index so the same
+            // (owner, model, document) is only ever watched under one source.
+            await db.schema.createTable('kubun_connector_projection_watch').ifNotExists().addColumn('owner_did', t.text, (col)=>col.notNull()).addColumn('connector_name', t.text, (col)=>col.notNull()).addColumn('model', t.text, (col)=>col.notNull()).addColumn('account_key', t.text, (col)=>col.notNull()).addColumn('source_id', t.text, (col)=>col.notNull()).addColumn('document_id', t.text, (col)=>col.notNull()).addColumn('verified_state_digest', t.text).addColumn('observed_winner_digest', t.text).addColumn('verification_status', t.text, (col)=>col.notNull()).addColumn('verification_attempts', 'integer', (col)=>col.notNull().defaultTo(0)).addColumn('repair_generation', 'integer', (col)=>col.notNull().defaultTo(0))// ms-epoch (Date.now) overflows int32 on Postgres; bigint holds it and
+            // the adapter's int8 parser reads it back as a JS number.
+            .addColumn('next_attempt_at', t.bigint)// Until when a verifier holds the current attempt (`claimWatchAttempt`);
+            // kept apart from `next_attempt_at`, which an ingress coalesce moves.
+            .addColumn('claimed_until', t.bigint)// As on the operation table: 'offline' when the pending attempt was
+            // deferred only for being offline.
+            .addColumn('defer_reason', t.text).addColumn('version', 'integer', (col)=>col.notNull().defaultTo(0))// ms-epoch (Date.now) overflows int32 on Postgres; bigint holds it and
+            // the adapter's int8 parser reads it back as a JS number.
+            .addColumn('created_at', t.bigint, (col)=>col.notNull()).addColumn('updated_at', t.bigint, (col)=>col.notNull()).addPrimaryKeyConstraint('kubun_connector_projection_watch_pkey', [
+                'owner_did',
+                'connector_name',
+                'model',
+                'account_key',
+                'source_id'
+            ]).execute();
+            // Ingress index: within one owner+model, a given provider document may
+            // only be watched under a single source. A write-back finalized for the
+            // document under another source supersedes the earlier row.
+            await db.schema.createIndex('uq_connector_watch_ingress').ifNotExists().on('kubun_connector_projection_watch').columns([
+                'owner_did',
+                'model',
+                'document_id'
+            ]).unique().execute();
+            // Document lookup: an applied write names only its document id
+            // (`getWatchByDocument`), which the ingress index cannot serve alone.
+            await db.schema.createIndex('idx_connector_watch_document').ifNotExists().on('kubun_connector_projection_watch').column('document_id').execute();
         },
         async down (db) {
+            await db.schema.dropTable('kubun_connector_projection_watch').ifExists().execute();
+            await db.schema.dropTable('kubun_connector_operation').ifExists().execute();
             await db.schema.dropTable('kubun_connector_pending_auth').ifExists().execute();
             await db.schema.dropTable('kubun_connector_sync_state').ifExists().execute();
         }

package/lib/tables.d.ts

@@ -8,6 +8,9 @@ export type ConnectorSyncStateTable = {
     status: string;
     error: string | null;
     lease_expires_at: number | null;
+    recovery: string | null;
+    version: number;
+    incarnation: string | null;
 };
 export type ConnectorSyncState = Selectable<ConnectorSyncStateTable>;
 export type ConnectorPendingAuthTable = {
@@ -34,7 +37,61 @@ export type ConnectorPendingAuthTable = {
     expires_at: number;
 };
 export type ConnectorPendingAuth = Selectable<ConnectorPendingAuthTable>;
+export type ConnectorOperationTable = {
+    caller_did: string;
+    operation_id: string;
+    node_key: string;
+    owner_did: string;
+    connector_name: string;
+    model: string;
+    op: string;
+    account_key: string;
+    target: string;
+    source_id: string | null;
+    deterministic_event_id: string;
+    payload_hash: string;
+    reissue_intent: string;
+    provider_etag: string | null;
+    document_id: string | null;
+    result_entity: string | null;
+    state: string;
+    attempts: number;
+    next_attempt_at: number | null;
+    last_kind: string | null;
+    last_http_status: number | null;
+    defer_reason: string | null;
+    send_marker: string | null;
+    version: number;
+    incarnation: string | null;
+    lease_token: string | null;
+    lease_expires_at: number | null;
+    created_at: number;
+    updated_at: number;
+};
+export type ConnectorOperation = Selectable<ConnectorOperationTable>;
+export type ConnectorProjectionWatchTable = {
+    owner_did: string;
+    connector_name: string;
+    model: string;
+    account_key: string;
+    source_id: string;
+    document_id: string;
+    verified_state_digest: string | null;
+    observed_winner_digest: string | null;
+    verification_status: string;
+    verification_attempts: number;
+    repair_generation: number;
+    next_attempt_at: number | null;
+    claimed_until: number | null;
+    defer_reason: string | null;
+    version: number;
+    created_at: number;
+    updated_at: number;
+};
+export type ConnectorProjectionWatch = Selectable<ConnectorProjectionWatchTable>;
 export type ConnectorTables = {
     kubun_connector_sync_state: ConnectorSyncStateTable;
     kubun_connector_pending_auth: ConnectorPendingAuthTable;
+    kubun_connector_operation: ConnectorOperationTable;
+    kubun_connector_projection_watch: ConnectorProjectionWatchTable;
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@kubun/store-connector",
-  "version": "0.17.0",
+  "version": "0.17.2",
   "keywords": [],
   "license": "see LICENSE.md",
   "sideEffects": false,
@@ -17,11 +17,11 @@
   "dependencies": {
     "@kubun/db": "^0.17.0",
     "@kubun/db-adapter": "^0.17.0",
-    "kysely": "^0.29.5"
+    "kysely": "^0.29.6"
   },
   "devDependencies": {
     "@kubun/db-better-sqlite": "^0.17.0",
-    "@kubun/db-postgres": "^0.17.0",
+    "@kubun/db-postgres": "^0.17.1",
     "@testcontainers/postgresql": "^12.1.0"
   },
   "publishConfig": {