@kubun/plugin-connector 0.12.1 → 0.13.0

package/lib/sync/workflow.js

@@ -0,0 +1,241 @@
+import { SyncEngine } from './engine.js';
+import { MAX_LOGGED_ERRORS } from './processor.js';
+/** Queue lane (and workflow name) for connector sync. */ export const CONNECTOR_SYNC_WORKFLOW = 'connector-sync';
+/** How many connector syncs may run at once on this peer. */ export const CONNECTOR_SYNC_CONCURRENCY = 4;
+/**
+ * Build the `connector-sync` workflow definition. One sync pass is a self-looping
+ * flow: `start` validates + seeds state, `batch` fetches+imports ONE provider
+ * batch per durable tick and advances the checkpoint, `finalize` marks idle. The
+ * workflow engine checkpoints the row and renews the lease between ticks, so the
+ * hand-rolled per-batch lease heartbeat is gone; the engine's crash-recovery
+ * re-drives `batch` from the persisted checkpoint.
+ *
+ * Each call returns a definition with its OWN in-process session map, so two
+ * definitions (two "boots" over one DB) do not share iterators — which is what
+ * makes crash-recovery re-open the provider from the checkpoint.
+ */ export function createConnectorSyncWorkflow(params) {
+    const { registry, credentialProvider, stateStore, syncEventEmitter, stores, mutateDocuments, logger } = params;
+    const sessions = new Map();
+    const sessionKey = (state)=>`${state.connectorName}:${state.ownerDID}`;
+    const effectiveBoundary = params.boundary ?? {
+        maxAge: 'P90D'
+    };
+    async function reportError(connectorName, ownerDID, checkpoint, lastSyncedAt, entityCount, err) {
+        const error = err instanceof Error ? err.message : String(err);
+        sessions.delete(`${connectorName}:${ownerDID}`);
+        await stateStore.set(connectorName, ownerDID, {
+            checkpoint,
+            lastSyncedAt,
+            entityCount,
+            status: 'error',
+            error
+        });
+        await syncEventEmitter.emitSyncEvent({
+            type: 'error',
+            connectorName,
+            error
+        });
+    }
+    const start = async (ctx)=>{
+        const { connectorName, ownerDID, full } = ctx.params;
+        try {
+            const connector = registry.get(connectorName);
+            if (connector == null) {
+                throw new Error(`Connector "${connectorName}" not found`);
+            }
+            const providerName = connector.auth.provider !== 'device' ? connector.auth.provider : null;
+            const credential = providerName ? await credentialProvider.get(providerName, ownerDID) : null;
+            if (providerName != null && credential == null) {
+                throw new Error(`Missing credential for connector "${connectorName}"`);
+            }
+            if (connector.serverProvider == null) {
+                throw new Error(`Connector "${connectorName}" has no server provider`);
+            }
+            const existing = await stateStore.get(connectorName, ownerDID);
+            const isIncremental = !full && existing?.checkpoint != null;
+            const checkpoint = existing?.checkpoint ?? null;
+            const lastSyncedAt = existing?.lastSyncedAt ?? new Date().toISOString();
+            const baseEntityCount = existing?.entityCount ?? 0;
+            await stateStore.set(connectorName, ownerDID, {
+                checkpoint,
+                lastSyncedAt,
+                entityCount: baseEntityCount,
+                status: 'syncing'
+            });
+            await syncEventEmitter.emitSyncEvent({
+                type: 'started',
+                connectorName
+            });
+            const state = {
+                connectorName,
+                ownerDID,
+                full,
+                phase: isIncremental ? 'incremental' : 'initial',
+                checkpoint,
+                baseEntityCount,
+                totalProcessed: 0,
+                totalFailed: 0,
+                batchNumber: 0,
+                startTime: Date.now(),
+                lastSyncedAt,
+                failedSample: []
+            };
+            return {
+                status: 'action',
+                state,
+                action: {
+                    name: 'batch'
+                }
+            };
+        } catch (err) {
+            await reportError(connectorName, ownerDID, null, new Date().toISOString(), 0, err);
+            throw err;
+        }
+    };
+    const batch = async (ctx)=>{
+        const state = ctx.state;
+        const key = sessionKey(state);
+        try {
+            let session = sessions.get(key);
+            if (session == null) {
+                // First batch, or a fresh process after a crash: (re)open the provider
+                // from the durable checkpoint. Re-processing an already-imported batch is
+                // idempotent (imports upsert by unique key).
+                const connector = registry.get(state.connectorName);
+                if (connector == null) {
+                    throw new Error(`Connector "${state.connectorName}" not found`);
+                }
+                const providerName = connector.auth.provider !== 'device' ? connector.auth.provider : null;
+                const credential = providerName ? await credentialProvider.get(providerName, state.ownerDID) : null;
+                if (connector.serverProvider == null) {
+                    throw new Error(`Connector "${state.connectorName}" has no server provider`);
+                }
+                const provider = connector.serverProvider({
+                    credential: credential ?? {
+                        accessToken: '',
+                        scopes: []
+                    },
+                    boundary: effectiveBoundary
+                });
+                const engine = new SyncEngine({
+                    stores,
+                    stateStore,
+                    connectorName: state.connectorName,
+                    clusters: connector.clusters,
+                    logger,
+                    mutateDocuments
+                });
+                const iterable = state.phase === 'incremental' && state.checkpoint != null ? provider.fetchChanges({
+                    boundary: effectiveBoundary,
+                    checkpoint: state.checkpoint,
+                    lastSyncedAt: state.lastSyncedAt ? new Date(state.lastSyncedAt) : undefined,
+                    signal: ctx.signal
+                }) : provider.fetchAll({
+                    boundary: effectiveBoundary,
+                    checkpoint: state.checkpoint ?? undefined,
+                    signal: ctx.signal
+                });
+                session = {
+                    engine,
+                    iterator: iterable[Symbol.asyncIterator]()
+                };
+                sessions.set(key, session);
+            }
+            const next = await session.iterator.next();
+            if (next.done || next.value == null) {
+                sessions.delete(key);
+                return {
+                    status: 'action',
+                    state,
+                    action: {
+                        name: 'finalize'
+                    }
+                };
+            }
+            const entityBatch = next.value;
+            const result = await session.engine.processBatch(entityBatch, state.ownerDID);
+            const totalProcessed = state.totalProcessed + result.created + result.updated + result.deleted;
+            const totalFailed = state.totalFailed + result.failed;
+            const failedSample = state.failedSample.slice();
+            for (const err of result.errors){
+                if (failedSample.length >= MAX_LOGGED_ERRORS) break;
+                failedSample.push(err);
+            }
+            const checkpoint = entityBatch.checkpoint != null ? entityBatch.checkpoint : state.checkpoint;
+            const lastSyncedAt = new Date().toISOString();
+            await stateStore.set(state.connectorName, state.ownerDID, {
+                checkpoint,
+                lastSyncedAt,
+                entityCount: state.baseEntityCount + totalProcessed,
+                status: 'syncing'
+            });
+            await syncEventEmitter.emitSyncEvent({
+                type: 'progress',
+                connectorName: state.connectorName,
+                entitiesProcessed: totalProcessed,
+                entitiesFailed: totalFailed
+            });
+            const nextState = {
+                ...state,
+                checkpoint,
+                totalProcessed,
+                totalFailed,
+                batchNumber: state.batchNumber + 1,
+                lastSyncedAt,
+                failedSample
+            };
+            return {
+                status: 'action',
+                state: nextState,
+                action: {
+                    name: 'batch'
+                }
+            };
+        } catch (err) {
+            await reportError(state.connectorName, state.ownerDID, state.checkpoint, state.lastSyncedAt, state.baseEntityCount + state.totalProcessed, err);
+            throw err;
+        }
+    };
+    const finalize = async (ctx)=>{
+        const state = ctx.state;
+        try {
+            const lastSyncedAt = new Date().toISOString();
+            await stateStore.set(state.connectorName, state.ownerDID, {
+                checkpoint: state.checkpoint,
+                lastSyncedAt,
+                entityCount: state.baseEntityCount + state.totalProcessed,
+                status: 'idle'
+            });
+            await syncEventEmitter.emitSyncEvent({
+                type: 'completed',
+                connectorName: state.connectorName,
+                totalProcessed: state.totalProcessed,
+                totalFailed: state.totalFailed,
+                duration: Date.now() - state.startTime,
+                ...state.failedSample.length > 0 ? {
+                    failedSample: state.failedSample
+                } : {}
+            });
+            return {
+                status: 'end',
+                state
+            };
+        } catch (err) {
+            await reportError(state.connectorName, state.ownerDID, state.checkpoint, state.lastSyncedAt, state.baseEntityCount + state.totalProcessed, err);
+            throw err;
+        }
+    };
+    return {
+        definition: {
+            name: CONNECTOR_SYNC_WORKFLOW,
+            initialAction: {
+                name: 'start'
+            },
+            handlers: {
+                start,
+                batch,
+                finalize
+            }
+        }
+    };
+}

package/lib/wrappable.d.ts

@@ -0,0 +1,26 @@
+import type { Identity, MethodRegistry } from '@kokuin/token';
+/**
+ * The spelling of a DID that can be encrypted to.
+ *
+ * `Identity` declares `id` alone, but a `did:peer:4` identity is a
+ * `MultiKeyIdentity` and carries `longForm` on the value. The short form
+ * resolves no document and so no agreement key — `deriveSharedSecret` refuses it
+ * outright, which is the good failure. The bad one is a key derived from the
+ * wrong branch, which throws nothing and yields ciphertext nobody can open.
+ *
+ * For `did:key` the two forms are the same string, so this is the identity's own
+ * `id` on every deployment Kubun ships today.
+ */
+export declare function wrappableDID(identity: Identity): string;
+/**
+ * Whether a key could be wrapped to this DID via the resolver-backed KEM.
+ *
+ * Asks the resolver rather than the spelling: this is the exact call a `did`
+ * factor makes when a credential key is minted, so the answer cannot drift from
+ * what the mint will do. A structural "is this a peer:4 short form?" test would,
+ * and would one day refuse a DID that works. A `did:kokuin` recipient resolves
+ * only when its resolver is passed in `methods`; without one it fails closed.
+ *
+ * The derived secret is discarded — resolving the recipient is the whole answer.
+ */
+export declare function canWrapTo(did: string, methods?: MethodRegistry): Promise<boolean>;

package/lib/wrappable.js

@@ -0,0 +1,36 @@
+import { deriveSharedSecretAsync } from '@kokuin/jwe';
+/**
+ * The spelling of a DID that can be encrypted to.
+ *
+ * `Identity` declares `id` alone, but a `did:peer:4` identity is a
+ * `MultiKeyIdentity` and carries `longForm` on the value. The short form
+ * resolves no document and so no agreement key — `deriveSharedSecret` refuses it
+ * outright, which is the good failure. The bad one is a key derived from the
+ * wrong branch, which throws nothing and yields ciphertext nobody can open.
+ *
+ * For `did:key` the two forms are the same string, so this is the identity's own
+ * `id` on every deployment Kubun ships today.
+ */ export function wrappableDID(identity) {
+    const longForm = identity.longForm;
+    return typeof longForm === 'string' ? longForm : identity.id;
+}
+/**
+ * Whether a key could be wrapped to this DID via the resolver-backed KEM.
+ *
+ * Asks the resolver rather than the spelling: this is the exact call a `did`
+ * factor makes when a credential key is minted, so the answer cannot drift from
+ * what the mint will do. A structural "is this a peer:4 short form?" test would,
+ * and would one day refuse a DID that works. A `did:kokuin` recipient resolves
+ * only when its resolver is passed in `methods`; without one it fails closed.
+ *
+ * The derived secret is discarded — resolving the recipient is the whole answer.
+ */ export async function canWrapTo(did, methods) {
+    try {
+        await deriveSharedSecretAsync(did, {
+            methods: methods ?? []
+        });
+        return true;
+    } catch  {
+        return false;
+    }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@kubun/plugin-connector",
-  "version": "0.12.1",
+  "version": "0.13.0",
   "license": "see LICENSE.md",
   "sideEffects": false,
   "type": "module",
@@ -14,8 +14,9 @@
     "LICENSE.md"
   ],
   "dependencies": {
-    "@kokuin/capability": "^0.2.2",
-    "@kokuin/token": "^0.4.0",
+    "@kokuin/capability": "^0.3.0",
+    "@kokuin/jwe": "^0.1.0",
+    "@kokuin/token": "^0.5.0",
     "@noble/hashes": "^2.3.0",
     "@sozai/async": "^0.2.1",
     "@sozai/codec": "^0.4.0",
@@ -23,25 +24,36 @@
     "@sozai/generator": "^0.2.0",
     "@sozai/runtime": "^0.1.0",
     "graphql": "^16.14.2",
-    "graphql-scalars": "^1.26.0",
+    "graphql-scalars": "^2.0.0",
     "kysely": "^0.29.5",
-    "@kubun/connector": "^0.12.1",
-    "@kubun/db": "^0.12.1",
-    "@kubun/engine": "^0.12.1",
-    "@kubun/hlc": "^0.12.0",
+    "@kubun/credential": "^0.13.0",
+    "@kubun/engine": "^0.13.0",
+    "@kubun/db": "^0.13.0",
     "@kubun/graphql": "^0.13.0",
-    "@kubun/id": "^0.12.0",
-    "@kubun/logger": "^0.12.0",
+    "@kubun/logger": "^0.13.0",
+    "@kubun/id": "^0.13.0",
     "@kubun/protocol": "^0.13.0",
-    "@kubun/store-connector": "^0.12.1",
-    "@kubun/store-delegation": "^0.12.1",
-    "@kubun/store-graph": "^0.13.0"
+    "@kubun/store-credential": "^0.13.0",
+    "@kubun/hlc": "^0.13.0",
+    "@kubun/store-graph": "^0.13.0",
+    "@kubun/store-connector": "^0.13.0",
+    "@kubun/connector": "^0.13.0",
+    "@kubun/store-delegation": "^0.13.0"
   },
   "devDependencies": {
+    "@kokuin/controller": "^0.1.0",
     "@testcontainers/postgresql": "^12.1.0",
     "get-port": "^7.2.0",
-    "@kubun/test-utils": "^0.12.0",
-    "@kubun/db-postgres": "^0.12.1"
+    "@kubun/db-postgres": "^0.13.0",
+    "@kubun/blob-backend": "^0.13.0",
+    "@kubun/connector-google-mail": "^0.13.0",
+    "@kubun/plugin-blob": "^0.13.0",
+    "@kubun/plugin-workflow": "^0.13.0",
+    "@kubun/store-blob": "^0.13.0",
+    "@kubun/models": "^0.13.0",
+    "@kubun/store-workflow": "^0.13.0",
+    "@kubun/test-utils": "^0.13.0",
+    "@kubun/store-controller": "^0.13.0"
   },
   "publishConfig": {
     "access": "public"