@kubun/plugin-connector 0.12.1 → 0.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/action.d.ts +7 -1
- package/lib/action.js +7 -4
- package/lib/api.d.ts +57 -6
- package/lib/api.js +110 -24
- package/lib/credential.d.ts +7 -1
- package/lib/credential.js +25 -3
- package/lib/index.d.ts +1 -3
- package/lib/index.js +138 -23
- package/lib/manager.d.ts +0 -2
- package/lib/manager.js +1 -2
- package/lib/oauth.js +31 -5
- package/lib/schema.d.ts +18 -0
- package/lib/schema.js +3 -1
- package/lib/sync/engine.d.ts +8 -3
- package/lib/sync/engine.js +9 -3
- package/lib/sync/orchestrate.d.ts +8 -1
- package/lib/sync/orchestrate.js +18 -13
- package/lib/sync/state.d.ts +0 -2
- package/lib/sync/state.js +0 -6
- package/lib/sync/workflow.d.ts +59 -0
- package/lib/sync/workflow.js +241 -0
- package/lib/wrappable.d.ts +26 -0
- package/lib/wrappable.js +36 -0
- package/package.json +27 -15
package/lib/index.js
CHANGED
|
@@ -1,4 +1,7 @@
|
|
|
1
|
+
import { isSigningIdentity } from '@kokuin/token';
|
|
2
|
+
import { createCredentialManager } from '@kubun/credential';
|
|
1
3
|
import { connectorStoreDefinition } from '@kubun/store-connector';
|
|
4
|
+
import { credentialStoreDefinition, getCredentialStore } from '@kubun/store-credential';
|
|
2
5
|
import { executeAction } from './action.js';
|
|
3
6
|
import { createConnectorAPI } from './api.js';
|
|
4
7
|
import { DBCredentialProvider } from './credential.js';
|
|
@@ -8,6 +11,8 @@ import { ConnectorRegistry } from './registry.js';
|
|
|
8
11
|
import { createConnectorSchemaExtension, subscribeToConnectorSyncEvents } from './schema.js';
|
|
9
12
|
import { orchestrateSync } from './sync/orchestrate.js';
|
|
10
13
|
import { DBSyncStateStore } from './sync/state.js';
|
|
14
|
+
import { CONNECTOR_SYNC_CONCURRENCY, CONNECTOR_SYNC_WORKFLOW, createConnectorSyncWorkflow } from './sync/workflow.js';
|
|
15
|
+
import { wrappableDID } from './wrappable.js';
|
|
11
16
|
import { createWriteGrantContext } from './write-grants.js';
|
|
12
17
|
// ---- Re-exports ----
|
|
13
18
|
export { executeAction } from './action.js';
|
|
@@ -19,9 +24,8 @@ export { SyncEngine } from './sync/engine.js';
|
|
|
19
24
|
export { orchestrateSync } from './sync/orchestrate.js';
|
|
20
25
|
export { EntityProcessor } from './sync/processor.js';
|
|
21
26
|
export { DBSyncStateStore } from './sync/state.js';
|
|
27
|
+
export { CONNECTOR_SYNC_CONCURRENCY, CONNECTOR_SYNC_WORKFLOW, createConnectorSyncWorkflow } from './sync/workflow.js';
|
|
22
28
|
export { DBCredentialProvider };
|
|
23
|
-
// ---- Plugin factory options ----
|
|
24
|
-
/** Default sync-lease TTL (5 minutes) — an in-flight sync heartbeats this before expiry. */ const DEFAULT_LEASE_TTL_MS = 300_000;
|
|
25
29
|
// ---- Connector state resolution ----
|
|
26
30
|
async function resolveConnectorState(connectorAPI, registry, credentialProvider, viewerDID, connectorName) {
|
|
27
31
|
const connector = registry.get(connectorName);
|
|
@@ -32,10 +36,15 @@ async function resolveConnectorState(connectorAPI, registry, credentialProvider,
|
|
|
32
36
|
let authenticated = false;
|
|
33
37
|
let hasWriteAccess = false;
|
|
34
38
|
let authExpiresAt = null;
|
|
39
|
+
let credentialUpdatedAt = null;
|
|
40
|
+
let credentialWriterDID = null;
|
|
35
41
|
if (connector.auth.provider !== 'device') {
|
|
36
42
|
const credential = await credentialProvider.get(connector.auth.provider, viewerDID);
|
|
37
43
|
if (credential != null) {
|
|
38
44
|
authenticated = true;
|
|
45
|
+
const provenance = await connectorAPI.getCredentialProvenance(connector.auth.provider, viewerDID);
|
|
46
|
+
credentialUpdatedAt = provenance?.updatedAt ?? null;
|
|
47
|
+
credentialWriterDID = provenance?.writerDID ?? null;
|
|
39
48
|
if (credential.expiresAt != null) {
|
|
40
49
|
authExpiresAt = credential.expiresAt.toISOString();
|
|
41
50
|
}
|
|
@@ -54,6 +63,8 @@ async function resolveConnectorState(connectorAPI, registry, credentialProvider,
|
|
|
54
63
|
authenticated,
|
|
55
64
|
hasWriteAccess,
|
|
56
65
|
authExpiresAt,
|
|
66
|
+
credentialUpdatedAt,
|
|
67
|
+
credentialWriterDID,
|
|
57
68
|
lastSyncedAt: null,
|
|
58
69
|
entityCount: null,
|
|
59
70
|
error: null
|
|
@@ -65,6 +76,8 @@ async function resolveConnectorState(connectorAPI, registry, credentialProvider,
|
|
|
65
76
|
authenticated,
|
|
66
77
|
hasWriteAccess,
|
|
67
78
|
authExpiresAt,
|
|
79
|
+
credentialUpdatedAt,
|
|
80
|
+
credentialWriterDID,
|
|
68
81
|
lastSyncedAt: syncState.lastSyncedAt ?? null,
|
|
69
82
|
entityCount: syncState.entityCount ?? null,
|
|
70
83
|
error: syncState.error ?? null
|
|
@@ -73,9 +86,17 @@ async function resolveConnectorState(connectorAPI, registry, credentialProvider,
|
|
|
73
86
|
// ---- Plugin factory ----
|
|
74
87
|
export function createConnectorPlugin(options) {
|
|
75
88
|
const providers = options.providers ?? [];
|
|
76
|
-
const leaseMs = options.syncLeaseTTL ?? DEFAULT_LEASE_TTL_MS;
|
|
77
89
|
return (params)=>{
|
|
78
90
|
params.db.register(connectorStoreDefinition);
|
|
91
|
+
params.db.register(credentialStoreDefinition);
|
|
92
|
+
// Refused at construction, not at the first write: every credential row this
|
|
93
|
+
// plugin stores carries an op signed by this identity, so an engine identity
|
|
94
|
+
// that cannot sign would fail on the first connector authorization instead
|
|
95
|
+
// of when the plugin was installed.
|
|
96
|
+
if (!isSigningIdentity(params.identity)) {
|
|
97
|
+
throw new Error('The connector plugin requires a SigningIdentity');
|
|
98
|
+
}
|
|
99
|
+
const identity = params.identity;
|
|
79
100
|
const logger = params.getLogger('connector');
|
|
80
101
|
// On a multi-tenant server the connector signs writes with the server
|
|
81
102
|
// identity, but imported documents must be owned by the account viewer. A
|
|
@@ -86,9 +107,49 @@ export function createConnectorPlugin(options) {
|
|
|
86
107
|
for (const connector of options.connectors){
|
|
87
108
|
registry.register(connector);
|
|
88
109
|
}
|
|
110
|
+
// Resolve the blob plugin's API lazily and once. Absent in a local-only app
|
|
111
|
+
// with no blob plugin — actions that need it must handle that. The action
|
|
112
|
+
// runs inside the request's mutation transaction, so the row insert must go
|
|
113
|
+
// through the request's provider (writeAttachmentTx), not the plugin's
|
|
114
|
+
// top-level db, which would open a second, deadlocking transaction.
|
|
115
|
+
let blobAPIPromise;
|
|
116
|
+
const getBlobAPI = ()=>{
|
|
117
|
+
if (blobAPIPromise == null) {
|
|
118
|
+
blobAPIPromise = params.engine.getAPI('blob').catch(()=>undefined);
|
|
119
|
+
}
|
|
120
|
+
return blobAPIPromise;
|
|
121
|
+
};
|
|
122
|
+
const serverWrappableDID = wrappableDID(params.identity);
|
|
123
|
+
// One manager per StoreProvider: the request-scoped provider is the
|
|
124
|
+
// mutation's transaction, and a manager built over the base provider would
|
|
125
|
+
// read outside it. The key cache is per-manager, which is the same scope the
|
|
126
|
+
// content key should have.
|
|
127
|
+
const managers = new WeakMap();
|
|
128
|
+
function getCredentialManager(stores) {
|
|
129
|
+
let manager = managers.get(stores);
|
|
130
|
+
if (manager == null) {
|
|
131
|
+
manager = getCredentialStore(stores).then((store)=>createCredentialManager({
|
|
132
|
+
store,
|
|
133
|
+
identity,
|
|
134
|
+
runtime: params.runtime,
|
|
135
|
+
hlc: params.hlc
|
|
136
|
+
}));
|
|
137
|
+
managers.set(stores, manager);
|
|
138
|
+
}
|
|
139
|
+
return manager;
|
|
140
|
+
}
|
|
141
|
+
// Controller resolvers for `did:kokuin:` wrapping recipients, scoped to the
|
|
142
|
+
// request's transaction provider so a resolve reads the mutation's tx rather
|
|
143
|
+
// than a second connection.
|
|
144
|
+
const getControllerMethods = (stores)=>[
|
|
145
|
+
params.controllerResolverFor(stores)
|
|
146
|
+
];
|
|
89
147
|
const connectorAPI = createConnectorAPI({
|
|
90
148
|
stores: params.db,
|
|
91
|
-
|
|
149
|
+
logger,
|
|
150
|
+
getCredentialManager,
|
|
151
|
+
serverWrappableDID,
|
|
152
|
+
getControllerMethods
|
|
92
153
|
});
|
|
93
154
|
const credentialProvider = new DBCredentialProvider({
|
|
94
155
|
api: connectorAPI,
|
|
@@ -110,6 +171,35 @@ export function createConnectorPlugin(options) {
|
|
|
110
171
|
credentialProvider,
|
|
111
172
|
connectorAPI
|
|
112
173
|
});
|
|
174
|
+
// The primary sync path: enqueue a durable `connector-sync` workflow whose
|
|
175
|
+
// engine owns the lease, checkpointing and crash-recovery. Built once here;
|
|
176
|
+
// registered lazily the first time the workflow API resolves.
|
|
177
|
+
const connectorSyncWorkflow = createConnectorSyncWorkflow({
|
|
178
|
+
registry,
|
|
179
|
+
credentialProvider,
|
|
180
|
+
stateStore,
|
|
181
|
+
syncEventEmitter: manager,
|
|
182
|
+
stores: params.db,
|
|
183
|
+
mutateDocuments: params.graph.mutateDocuments,
|
|
184
|
+
boundary: options.defaults?.boundary,
|
|
185
|
+
logger
|
|
186
|
+
});
|
|
187
|
+
let workflowAPIPromise;
|
|
188
|
+
const getWorkflowAPI = ()=>{
|
|
189
|
+
if (workflowAPIPromise == null) {
|
|
190
|
+
workflowAPIPromise = params.engine.getAPI('workflow').then((api)=>{
|
|
191
|
+
api.defineQueue(CONNECTOR_SYNC_WORKFLOW, {
|
|
192
|
+
concurrency: CONNECTOR_SYNC_CONCURRENCY
|
|
193
|
+
});
|
|
194
|
+
api.register(connectorSyncWorkflow.definition);
|
|
195
|
+
return api;
|
|
196
|
+
}).catch(()=>undefined);
|
|
197
|
+
}
|
|
198
|
+
return workflowAPIPromise;
|
|
199
|
+
};
|
|
200
|
+
// Process-local dedup for the no-workflow-plugin fallback path only (the
|
|
201
|
+
// workflow singleton guards the primary path).
|
|
202
|
+
const orchestrateInFlight = new Set();
|
|
113
203
|
return {
|
|
114
204
|
name: 'connector',
|
|
115
205
|
schemaExtension: (config)=>createConnectorSchemaExtension({
|
|
@@ -123,7 +213,10 @@ export function createConnectorPlugin(options) {
|
|
|
123
213
|
// request's StoreProvider (transactional during mutations).
|
|
124
214
|
const requestAPI = createConnectorAPI({
|
|
125
215
|
stores,
|
|
126
|
-
|
|
216
|
+
logger,
|
|
217
|
+
getCredentialManager,
|
|
218
|
+
serverWrappableDID,
|
|
219
|
+
getControllerMethods
|
|
127
220
|
});
|
|
128
221
|
const requestCredentialProvider = new DBCredentialProvider({
|
|
129
222
|
api: requestAPI,
|
|
@@ -145,24 +238,41 @@ export function createConnectorPlugin(options) {
|
|
|
145
238
|
connectorName: args.connector
|
|
146
239
|
};
|
|
147
240
|
}
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
241
|
+
const connectorName = args.connector;
|
|
242
|
+
const ownerDID = ctx.viewerDID;
|
|
243
|
+
const full = args.full ?? false;
|
|
244
|
+
// Defer to after the mutation commits — both the workflow enqueue
|
|
245
|
+
// (a store insert) and the fallback background run use the top-level
|
|
246
|
+
// db, which would deadlock if the mutation still held the connection.
|
|
151
247
|
stores.onCommit(()=>{
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
248
|
+
void (async ()=>{
|
|
249
|
+
const workflowAPI = await getWorkflowAPI();
|
|
250
|
+
if (workflowAPI != null) {
|
|
251
|
+
await workflowAPI.enqueue(CONNECTOR_SYNC_WORKFLOW, {
|
|
252
|
+
connectorName,
|
|
253
|
+
ownerDID,
|
|
254
|
+
full
|
|
255
|
+
}, {
|
|
256
|
+
singletonKey: `${connectorName}:${ownerDID}`
|
|
257
|
+
});
|
|
258
|
+
return;
|
|
259
|
+
}
|
|
260
|
+
// No workflow plugin (local-only app): run inline.
|
|
261
|
+
await orchestrateSync({
|
|
262
|
+
connectorName,
|
|
263
|
+
full,
|
|
264
|
+
registry,
|
|
265
|
+
syncEventEmitter: manager,
|
|
266
|
+
stateStore,
|
|
267
|
+
credentialProvider,
|
|
268
|
+
ownerDID,
|
|
269
|
+
stores: params.db,
|
|
270
|
+
boundary: options.defaults?.boundary,
|
|
271
|
+
inFlight: orchestrateInFlight,
|
|
272
|
+
logger,
|
|
273
|
+
mutateDocuments: params.graph.mutateDocuments
|
|
274
|
+
});
|
|
275
|
+
})().catch(()=>{});
|
|
166
276
|
});
|
|
167
277
|
return {
|
|
168
278
|
status: 'STARTED',
|
|
@@ -172,6 +282,10 @@ export function createConnectorPlugin(options) {
|
|
|
172
282
|
subscribeToSyncEvents: (connector)=>subscribeToConnectorSyncEvents(manager.syncEvents, connector),
|
|
173
283
|
executeAction: async (args, writeDocument)=>{
|
|
174
284
|
try {
|
|
285
|
+
// Bind the blob write to THIS request's transactional provider so
|
|
286
|
+
// the row insert joins the mutation transaction.
|
|
287
|
+
const blob = await getBlobAPI();
|
|
288
|
+
const writeAttachment = blob != null ? (input, options)=>blob.writeAttachmentTx(stores, input, options) : undefined;
|
|
175
289
|
const result = await executeAction({
|
|
176
290
|
connector: args.connector,
|
|
177
291
|
action: args.action,
|
|
@@ -184,7 +298,8 @@ export function createConnectorPlugin(options) {
|
|
|
184
298
|
credentialProvider: requestCredentialProvider,
|
|
185
299
|
ownerDID: ctx.viewerDID,
|
|
186
300
|
logger,
|
|
187
|
-
writeDocument
|
|
301
|
+
writeDocument,
|
|
302
|
+
writeAttachment
|
|
188
303
|
});
|
|
189
304
|
return {
|
|
190
305
|
documentID: result.documentID,
|
package/lib/manager.d.ts
CHANGED
|
@@ -21,7 +21,6 @@ export type ConnectorManagerParams = {
|
|
|
21
21
|
logger: Logger;
|
|
22
22
|
defaults?: {
|
|
23
23
|
boundary?: SyncBoundary;
|
|
24
|
-
pollingInterval?: string;
|
|
25
24
|
};
|
|
26
25
|
};
|
|
27
26
|
export declare class ConnectorManager {
|
|
@@ -29,7 +28,6 @@ export declare class ConnectorManager {
|
|
|
29
28
|
constructor(params: ConnectorManagerParams);
|
|
30
29
|
get defaults(): {
|
|
31
30
|
boundary: SyncBoundary;
|
|
32
|
-
pollingInterval: string;
|
|
33
31
|
};
|
|
34
32
|
activateForGraph(graphID: string, connectorNames: Array<string>): void;
|
|
35
33
|
deactivateForGraph(graphID: string, connectorNames: Array<string>): void;
|
package/lib/manager.js
CHANGED
package/lib/oauth.js
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { sha256 } from '@noble/hashes/sha2.js';
|
|
2
2
|
import { fromUTF, toB64U } from '@sozai/codec';
|
|
3
|
+
import { canWrapTo } from './wrappable.js';
|
|
3
4
|
// Pending authorization records self-expire after this window; a callback
|
|
4
5
|
// presenting an older state is rejected and the row is swept on the next start.
|
|
5
6
|
const PENDING_AUTH_TTL_MS = 10 * 60 * 1000;
|
|
@@ -31,6 +32,19 @@ export class OAuthService {
|
|
|
31
32
|
if (provider.clientID == null) {
|
|
32
33
|
throw new Error(`OAuth provider "${args.provider}" is missing clientID`);
|
|
33
34
|
}
|
|
35
|
+
// Carried from here to the callback: the client knows its own long form, and
|
|
36
|
+
// the callback is the only moment at which the key gets minted.
|
|
37
|
+
const ownerWrappableDID = args.viewerWrappableDID ?? ownerDID;
|
|
38
|
+
// Asked here rather than at the mint, which is where it used to fail. By
|
|
39
|
+
// then the pending row is consumed and the authorization code is spent, so
|
|
40
|
+
// the provider has issued a real token pair that the failure discards and
|
|
41
|
+
// the user has to run the whole dance again.
|
|
42
|
+
if (!await canWrapTo(ownerWrappableDID)) {
|
|
43
|
+
throw new Error(`Cannot start OAuth for "${args.provider}": nothing can be encrypted to "${ownerWrappableDID}", so the credential's owner could never read it. Pass \`viewerWrappableDID\` — for a did:peer:4 viewer that is the long form, which carries the agreement key.`);
|
|
44
|
+
}
|
|
45
|
+
// Carried to the mint like the owner's wrappable DID; it becomes a
|
|
46
|
+
// recovery recipient later.
|
|
47
|
+
const controllerWrappableDID = args.controllerWrappableDID ?? undefined;
|
|
34
48
|
const requestWriteAccess = args.requestWriteAccess === true;
|
|
35
49
|
const scopes = new Set(provider.baseScopes ?? []);
|
|
36
50
|
const connectorNames = args.connectors ?? this.#registry.list();
|
|
@@ -56,7 +70,13 @@ export class OAuthService {
|
|
|
56
70
|
state,
|
|
57
71
|
codeVerifier,
|
|
58
72
|
provider: args.provider,
|
|
73
|
+
// Not normalized: nothing looks this row up by DID — it is keyed on
|
|
74
|
+
// `state` — and the `ConnectorAPI` boundary normalizes when it files the
|
|
75
|
+
// credential. Folding it here would put a normalized DID one line from a
|
|
76
|
+
// wrappable one that must not be.
|
|
59
77
|
ownerDID,
|
|
78
|
+
ownerWrappableDID,
|
|
79
|
+
controllerWrappableDID,
|
|
60
80
|
createdAt: Date.now()
|
|
61
81
|
});
|
|
62
82
|
await api.deleteExpiredPendingAuth(Date.now() - PENDING_AUTH_TTL_MS);
|
|
@@ -123,11 +143,17 @@ export class OAuthService {
|
|
|
123
143
|
const scopes = tokenData.scope?.split(' ') ?? [];
|
|
124
144
|
// Bind the credential to the owner that initiated the flow, not the ambient
|
|
125
145
|
// viewer completing the callback, so a stolen code cannot land under another DID.
|
|
126
|
-
await (credentialProvider ?? this.#credentialProvider).set(
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
146
|
+
await (credentialProvider ?? this.#credentialProvider).set({
|
|
147
|
+
providerName: args.provider,
|
|
148
|
+
ownerDID: pending.ownerDID,
|
|
149
|
+
credential: {
|
|
150
|
+
accessToken: tokenData.access_token,
|
|
151
|
+
refreshToken: tokenData.refresh_token,
|
|
152
|
+
expiresAt: tokenData.expires_in != null ? new Date(Date.now() + tokenData.expires_in * 1000) : undefined,
|
|
153
|
+
scopes
|
|
154
|
+
},
|
|
155
|
+
ownerWrappableDID: pending.ownerWrappableDID ?? pending.ownerDID,
|
|
156
|
+
controllerWrappableDID: pending.controllerWrappableDID ?? undefined
|
|
131
157
|
});
|
|
132
158
|
return {
|
|
133
159
|
providerName: args.provider,
|
package/lib/schema.d.ts
CHANGED
|
@@ -13,6 +13,8 @@ export type ConnectorState = {
|
|
|
13
13
|
authenticated: boolean;
|
|
14
14
|
hasWriteAccess: boolean;
|
|
15
15
|
authExpiresAt: string | null;
|
|
16
|
+
credentialUpdatedAt: string | null;
|
|
17
|
+
credentialWriterDID: string | null;
|
|
16
18
|
lastSyncedAt: string | null;
|
|
17
19
|
entityCount: number | null;
|
|
18
20
|
error: string | null;
|
|
@@ -25,6 +27,22 @@ export type StartConnectorAuthInput = {
|
|
|
25
27
|
redirectURL: string;
|
|
26
28
|
connectors?: Array<string> | null;
|
|
27
29
|
requestWriteAccess?: boolean | null;
|
|
30
|
+
/**
|
|
31
|
+
* The viewer's own DID in a form that can be encrypted to.
|
|
32
|
+
*
|
|
33
|
+
* Supplied rather than derived: the viewer DID reaching the server is a
|
|
34
|
+
* mutation's `iss`, which for `did:peer:4` is the long form only on first
|
|
35
|
+
* contact with an audience and the short form after. The client always knows
|
|
36
|
+
* its own long form, and handing it over is what keeps a peer:4 owner able to
|
|
37
|
+
* read the credential minted for them.
|
|
38
|
+
*/
|
|
39
|
+
viewerWrappableDID?: string | null;
|
|
40
|
+
/**
|
|
41
|
+
* A recovery recipient's DID in a form that can be encrypted to, carried
|
|
42
|
+
* through to the mint like `viewerWrappableDID`. Not yet turned into a
|
|
43
|
+
* wrapping.
|
|
44
|
+
*/
|
|
45
|
+
controllerWrappableDID?: string | null;
|
|
28
46
|
};
|
|
29
47
|
export type StartConnectorAuthOutput = {
|
|
30
48
|
url: string;
|
package/lib/schema.js
CHANGED
|
@@ -111,6 +111,8 @@ type Connector implements Node {
|
|
|
111
111
|
authenticated: Boolean!
|
|
112
112
|
hasWriteAccess: Boolean!
|
|
113
113
|
authExpiresAt: String
|
|
114
|
+
credentialUpdatedAt: String
|
|
115
|
+
credentialWriterDID: String
|
|
114
116
|
lastSyncedAt: String
|
|
115
117
|
entityCount: Int
|
|
116
118
|
error: String
|
|
@@ -163,7 +165,7 @@ extend type Query {
|
|
|
163
165
|
}
|
|
164
166
|
|
|
165
167
|
extend type Mutation {
|
|
166
|
-
startConnectorAuth(provider: String!, redirectURL: String!, connectors: [String!], requestWriteAccess: Boolean): StartConnectorAuthResult!
|
|
168
|
+
startConnectorAuth(provider: String!, redirectURL: String!, connectors: [String!], requestWriteAccess: Boolean, viewerWrappableDID: String, controllerWrappableDID: String): StartConnectorAuthResult!
|
|
167
169
|
completeConnectorAuth(provider: String!, code: String!, redirectURL: String!, state: String!): CompleteConnectorAuthResult!
|
|
168
170
|
syncConnector(connector: String!, full: Boolean): SyncTriggerResult!
|
|
169
171
|
grantConnectorWriteCapability(connector: String!, tokens: [String!]!): Boolean!
|
package/lib/sync/engine.d.ts
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
import type { DataProvider, SyncBoundary, SyncEvent, SyncStateStore } from '@kubun/connector';
|
|
1
|
+
import type { DataProvider, EntityBatch, SyncBoundary, SyncEvent, SyncStateStore } from '@kubun/connector';
|
|
2
2
|
import type { StoreProvider } from '@kubun/db';
|
|
3
3
|
import type { Logger } from '@kubun/logger';
|
|
4
4
|
import type { ClustersRecord } from '@kubun/protocol';
|
|
5
|
-
import { type MutateDocuments } from './processor.js';
|
|
5
|
+
import { type MutateDocuments, type ProcessBatchResult } from './processor.js';
|
|
6
6
|
export type SyncEngineParams = {
|
|
7
7
|
stores: StoreProvider;
|
|
8
8
|
stateStore: SyncStateStore;
|
|
@@ -18,11 +18,16 @@ export type RunSyncParams = {
|
|
|
18
18
|
boundary: SyncBoundary;
|
|
19
19
|
full?: boolean;
|
|
20
20
|
signal?: AbortSignal;
|
|
21
|
-
leaseMs: number;
|
|
22
21
|
};
|
|
23
22
|
export declare class SyncEngine {
|
|
24
23
|
#private;
|
|
25
24
|
constructor(params: SyncEngineParams);
|
|
26
25
|
on(event: 'sync', callback: (event: SyncEvent) => void): () => void;
|
|
26
|
+
/**
|
|
27
|
+
* Fetch + import a single provider batch. The workflow-driven sync path calls
|
|
28
|
+
* this once per durable tick, reusing the engine's model lookup and processor
|
|
29
|
+
* cache; {@link run} calls it inline for the no-workflow-plugin fallback.
|
|
30
|
+
*/
|
|
31
|
+
processBatch(batch: EntityBatch, ownerDID: string): Promise<ProcessBatchResult>;
|
|
27
32
|
run(params: RunSyncParams): Promise<void>;
|
|
28
33
|
}
|
package/lib/sync/engine.js
CHANGED
|
@@ -62,8 +62,16 @@ export class SyncEngine {
|
|
|
62
62
|
this.#processors.set(modelID, processor);
|
|
63
63
|
return processor;
|
|
64
64
|
}
|
|
65
|
+
/**
|
|
66
|
+
* Fetch + import a single provider batch. The workflow-driven sync path calls
|
|
67
|
+
* this once per durable tick, reusing the engine's model lookup and processor
|
|
68
|
+
* cache; {@link run} calls it inline for the no-workflow-plugin fallback.
|
|
69
|
+
*/ async processBatch(batch, ownerDID) {
|
|
70
|
+
const processor = this.#getProcessor(batch.modelID, ownerDID);
|
|
71
|
+
return processor.processBatch(batch);
|
|
72
|
+
}
|
|
65
73
|
async run(params) {
|
|
66
|
-
const { provider, ownerDID, boundary, full, signal
|
|
74
|
+
const { provider, ownerDID, boundary, full, signal } = params;
|
|
67
75
|
const startTime = Date.now();
|
|
68
76
|
const existingState = await this.#stateStore.get(this.#connectorName, ownerDID);
|
|
69
77
|
const isIncremental = !full && existingState?.checkpoint != null;
|
|
@@ -119,8 +127,6 @@ export class SyncEngine {
|
|
|
119
127
|
entityCount: (existingState?.entityCount ?? 0) + totalProcessed,
|
|
120
128
|
status: 'syncing'
|
|
121
129
|
});
|
|
122
|
-
// Heartbeat the lease so a long-running sync is not reclaimed mid-flight.
|
|
123
|
-
await this.#stateStore.renewSync(this.#connectorName, ownerDID, leaseMs);
|
|
124
130
|
}
|
|
125
131
|
const duration = Date.now() - startTime;
|
|
126
132
|
await this.#stateStore.set(this.#connectorName, ownerDID, {
|
|
@@ -21,10 +21,17 @@ export type OrchestrateSyncParams = {
|
|
|
21
21
|
ownerDID: string;
|
|
22
22
|
stores: StoreProvider;
|
|
23
23
|
boundary?: SyncBoundary;
|
|
24
|
-
leaseMs: number;
|
|
25
24
|
logger: Logger;
|
|
26
25
|
/** Engine-signed write helper routing imports through the mutation pipeline. */
|
|
27
26
|
mutateDocuments?: MutateDocuments;
|
|
27
|
+
/**
|
|
28
|
+
* Process-local guard against a concurrent duplicate pass for the same
|
|
29
|
+
* (connector, owner). This is the fallback path taken only when no workflow
|
|
30
|
+
* plugin is installed (a local-only app); the workflow engine's singleton is
|
|
31
|
+
* the guard on the primary path. Omitted = no guard (double-runs are benign:
|
|
32
|
+
* imports upsert by unique key).
|
|
33
|
+
*/
|
|
34
|
+
inFlight?: Set<string>;
|
|
28
35
|
};
|
|
29
36
|
export type OrchestrateSyncResult = {
|
|
30
37
|
status: 'started' | 'already_syncing' | 'error';
|
package/lib/sync/orchestrate.js
CHANGED
|
@@ -1,35 +1,41 @@
|
|
|
1
1
|
import { SyncEngine } from './engine.js';
|
|
2
2
|
export async function orchestrateSync(params) {
|
|
3
|
-
const { connectorName, full, registry, syncEventEmitter, stateStore, credentialProvider, ownerDID, stores,
|
|
3
|
+
const { connectorName, full, registry, syncEventEmitter, stateStore, credentialProvider, ownerDID, stores, inFlight, logger } = params;
|
|
4
4
|
const connector = registry.get(connectorName);
|
|
5
5
|
if (connector == null) {
|
|
6
6
|
return {
|
|
7
7
|
status: 'error'
|
|
8
8
|
};
|
|
9
9
|
}
|
|
10
|
+
// Reserve the (connector, owner) synchronously — before any await — so two
|
|
11
|
+
// concurrent triggers cannot both pass the check. Released once the run
|
|
12
|
+
// settles (or here on any early bail).
|
|
13
|
+
const guardKey = `${connectorName}:${ownerDID}`;
|
|
14
|
+
if (inFlight != null) {
|
|
15
|
+
if (inFlight.has(guardKey)) {
|
|
16
|
+
return {
|
|
17
|
+
status: 'already_syncing'
|
|
18
|
+
};
|
|
19
|
+
}
|
|
20
|
+
inFlight.add(guardKey);
|
|
21
|
+
}
|
|
22
|
+
const release = ()=>inFlight?.delete(guardKey);
|
|
10
23
|
// Get credentials for OAuth connectors
|
|
11
24
|
const providerName = connector.auth.provider !== 'device' ? connector.auth.provider : null;
|
|
12
25
|
const credential = providerName ? await credentialProvider.get(providerName, ownerDID) : null;
|
|
13
26
|
if (providerName != null && credential == null) {
|
|
27
|
+
release();
|
|
14
28
|
return {
|
|
15
29
|
status: 'error'
|
|
16
30
|
};
|
|
17
31
|
}
|
|
18
32
|
// Ensure server provider is available
|
|
19
33
|
if (connector.serverProvider == null) {
|
|
34
|
+
release();
|
|
20
35
|
return {
|
|
21
36
|
status: 'error'
|
|
22
37
|
};
|
|
23
38
|
}
|
|
24
|
-
// Atomically claim the sync lease. This happens after the validation paths
|
|
25
|
-
// above so a failed claim never leaves a live lease with no running sync.
|
|
26
|
-
// A stale/expired lease (e.g. from a crashed run) is reclaimed automatically.
|
|
27
|
-
const claimed = await stateStore.claimSync(connectorName, ownerDID, leaseMs);
|
|
28
|
-
if (!claimed) {
|
|
29
|
-
return {
|
|
30
|
-
status: 'already_syncing'
|
|
31
|
-
};
|
|
32
|
-
}
|
|
33
39
|
const effectiveBoundary = params.boundary ?? {
|
|
34
40
|
maxAge: 'P90D'
|
|
35
41
|
};
|
|
@@ -85,15 +91,14 @@ export async function orchestrateSync(params) {
|
|
|
85
91
|
provider,
|
|
86
92
|
ownerDID,
|
|
87
93
|
boundary: effectiveBoundary,
|
|
88
|
-
full
|
|
89
|
-
leaseMs
|
|
94
|
+
full
|
|
90
95
|
}).catch(async (err)=>{
|
|
91
96
|
await syncEventEmitter.emitSyncEvent({
|
|
92
97
|
type: 'error',
|
|
93
98
|
connectorName,
|
|
94
99
|
error: err instanceof Error ? err.message : String(err)
|
|
95
100
|
});
|
|
96
|
-
});
|
|
101
|
+
}).finally(release);
|
|
97
102
|
return {
|
|
98
103
|
status: 'started'
|
|
99
104
|
};
|
package/lib/sync/state.d.ts
CHANGED
|
@@ -12,8 +12,6 @@ export declare class DBSyncStateStore implements SyncStateStore {
|
|
|
12
12
|
});
|
|
13
13
|
get(connectorName: string, ownerDID: string): Promise<SyncState | null>;
|
|
14
14
|
set(connectorName: string, ownerDID: string, state: SyncState): Promise<void>;
|
|
15
|
-
claimSync(connectorName: string, ownerDID: string, leaseMs: number): Promise<boolean>;
|
|
16
|
-
renewSync(connectorName: string, ownerDID: string, leaseMs: number): Promise<void>;
|
|
17
15
|
delete(connectorName: string, ownerDID: string): Promise<void>;
|
|
18
16
|
listForConnector(connectorName: string): Promise<Array<SyncStateEntry>>;
|
|
19
17
|
}
|
package/lib/sync/state.js
CHANGED
|
@@ -31,12 +31,6 @@
|
|
|
31
31
|
error: state.error ?? null
|
|
32
32
|
});
|
|
33
33
|
}
|
|
34
|
-
async claimSync(connectorName, ownerDID, leaseMs) {
|
|
35
|
-
return this.#api.claimSync(connectorName, ownerDID, leaseMs);
|
|
36
|
-
}
|
|
37
|
-
async renewSync(connectorName, ownerDID, leaseMs) {
|
|
38
|
-
await this.#api.renewSync(connectorName, ownerDID, leaseMs);
|
|
39
|
-
}
|
|
40
34
|
async delete(connectorName, ownerDID) {
|
|
41
35
|
await this.#api.deleteSyncState(connectorName, ownerDID);
|
|
42
36
|
}
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
import type { CredentialProvider, SyncBoundary, SyncStateStore } from '@kubun/connector';
|
|
2
|
+
import type { StoreProvider } from '@kubun/db';
|
|
3
|
+
import type { Logger } from '@kubun/logger';
|
|
4
|
+
import type { ConnectorRegistry } from '../registry.js';
|
|
5
|
+
import type { SyncEventEmitter } from './orchestrate.js';
|
|
6
|
+
import { type MutateDocuments } from './processor.js';
|
|
7
|
+
/** Queue lane (and workflow name) for connector sync. */
|
|
8
|
+
export declare const CONNECTOR_SYNC_WORKFLOW = "connector-sync";
|
|
9
|
+
/** How many connector syncs may run at once on this peer. */
|
|
10
|
+
export declare const CONNECTOR_SYNC_CONCURRENCY = 4;
|
|
11
|
+
type HandlerContext = {
|
|
12
|
+
state: Record<string, unknown>;
|
|
13
|
+
params?: unknown;
|
|
14
|
+
signal?: AbortSignal;
|
|
15
|
+
};
|
|
16
|
+
type HandlerResult = {
|
|
17
|
+
status: 'action';
|
|
18
|
+
state: Record<string, unknown>;
|
|
19
|
+
action: {
|
|
20
|
+
name: string;
|
|
21
|
+
};
|
|
22
|
+
} | {
|
|
23
|
+
status: 'end';
|
|
24
|
+
state: Record<string, unknown>;
|
|
25
|
+
};
|
|
26
|
+
type ConnectorSyncHandler = (ctx: HandlerContext) => Promise<HandlerResult>;
|
|
27
|
+
export type ConnectorSyncWorkflowDefinition = {
|
|
28
|
+
name: string;
|
|
29
|
+
initialAction: {
|
|
30
|
+
name: string;
|
|
31
|
+
};
|
|
32
|
+
handlers: Record<string, ConnectorSyncHandler>;
|
|
33
|
+
};
|
|
34
|
+
export type ConnectorSyncWorkflowParams = {
|
|
35
|
+
registry: ConnectorRegistry;
|
|
36
|
+
credentialProvider: CredentialProvider;
|
|
37
|
+
stateStore: SyncStateStore;
|
|
38
|
+
syncEventEmitter: SyncEventEmitter;
|
|
39
|
+
stores: StoreProvider;
|
|
40
|
+
mutateDocuments?: MutateDocuments;
|
|
41
|
+
boundary?: SyncBoundary;
|
|
42
|
+
logger: Logger;
|
|
43
|
+
};
|
|
44
|
+
/**
|
|
45
|
+
* Build the `connector-sync` workflow definition. One sync pass is a self-looping
|
|
46
|
+
* flow: `start` validates + seeds state, `batch` fetches+imports ONE provider
|
|
47
|
+
* batch per durable tick and advances the checkpoint, `finalize` marks idle. The
|
|
48
|
+
* workflow engine checkpoints the row and renews the lease between ticks, so the
|
|
49
|
+
* hand-rolled per-batch lease heartbeat is gone; the engine's crash-recovery
|
|
50
|
+
* re-drives `batch` from the persisted checkpoint.
|
|
51
|
+
*
|
|
52
|
+
* Each call returns a definition with its OWN in-process session map, so two
|
|
53
|
+
* definitions (two "boots" over one DB) do not share iterators — which is what
|
|
54
|
+
* makes crash-recovery re-open the provider from the checkpoint.
|
|
55
|
+
*/
|
|
56
|
+
export declare function createConnectorSyncWorkflow(params: ConnectorSyncWorkflowParams): {
|
|
57
|
+
definition: ConnectorSyncWorkflowDefinition;
|
|
58
|
+
};
|
|
59
|
+
export {};
|