@kubun/plugin-connector 0.8.0 → 0.8.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/action.d.ts +2 -2
- package/lib/action.js +1 -1
- package/lib/api.d.ts +2 -2
- package/lib/api.js +1 -1
- package/lib/index.js +1 -1
- package/lib/oauth.d.ts +1 -1
- package/lib/oauth.js +1 -1
- package/lib/schema.d.ts +15 -7
- package/lib/schema.js +7 -7
- package/lib/sync/engine.d.ts +2 -2
- package/lib/sync/engine.js +1 -1
- package/lib/sync/orchestrate.d.ts +2 -2
- package/lib/sync/orchestrate.js +1 -1
- package/lib/sync/processor.d.ts +2 -2
- package/lib/sync/processor.js +1 -1
- package/package.json +10 -8
- package/lib/migration.d.ts +0 -3
- package/lib/migration.js +0 -1
package/lib/action.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { CredentialProvider, EntityRecord } from '@kubun/connector';
|
|
2
|
-
import type {
|
|
2
|
+
import type { StoreProvider } from '@kubun/db';
|
|
3
3
|
import type { ConnectorRegistry } from './registry.js';
|
|
4
4
|
export type ExecuteActionParams = {
|
|
5
5
|
connector: string;
|
|
@@ -13,7 +13,7 @@ export type ExecuteActionResult = {
|
|
|
13
13
|
documentID: string | null;
|
|
14
14
|
};
|
|
15
15
|
export type ExecuteActionDeps = {
|
|
16
|
-
|
|
16
|
+
stores: StoreProvider;
|
|
17
17
|
registry: ConnectorRegistry;
|
|
18
18
|
credentialProvider: CredentialProvider;
|
|
19
19
|
ownerDID: string;
|
package/lib/action.js
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
import{DocumentID as e,DocumentModelID as r}from"@kubun/id";import{
|
|
1
|
+
import{DocumentID as e,DocumentModelID as r}from"@kubun/id";import{getGraphStore as o}from"@kubun/store-graph";import{EntityProcessor as t}from"./sync/processor.js";export async function executeAction(n,l){let c,i,{stores:a,registry:u,credentialProvider:s,ownerDID:d}=l,{connector:f,action:p,model:w,input:m,sourceID:E}=n,h=u.get(f);if(null==h)throw Error(`Connector "${f}" not found`);if(null==h.actionHandler)throw Error(`Connector "${f}" does not support actions`);if(null==h.actions?.find(e=>e.name===p))throw Error(`Action "${p}" not found on connector "${f}"`);let S=h.auth.provider,$=await s.get(S,d);if(null==$)throw Error(`No credential found for provider "${S}"`);let I=h.auth;if(null!=I.writeScopes&&!I.writeScopes.every(e=>$.scopes.includes(e))){let e=Error("INSUFFICIENT_SCOPES");throw e.code="INSUFFICIENT_SCOPES",e.requiredScopes=I.writeScopes,e}for(let e of Object.values(h.clusters)){for(let[r,o]of Object.entries(e.record))if(e.models[o].name===w){c=r;break}if(null!=c)break}if(null==c)throw Error(`Model "${w}" not found in connector "${f}" clusters`);let b=m;if("update"===p&&null!=E){let t=new TextEncoder().encode(`${f}:${E}`),n=e.create(r.fromString(c),d,t),l=await o(a),i=await l.getDocument(n);i?.data!=null&&(b={...i.data,...m})}let C=h.actionHandler({credential:$});if("create"===p)i=await C.create(w,b);else if("update"===p){if(null==E)throw Error("sourceID is required for update actions");i=await C.update(w,E,b)}else throw Error(`Unsupported action: ${p}`);let g={};for(let e of Object.values(h.clusters))if(null!=e.record[c]){g=e.models[e.record[c]]?.fieldsMeta??{};break}let k=new t({stores:a,modelID:c,ownerDID:d,connectorName:f,edgeFields:g,clusters:h.clusters}),v=await k.processBatch({entities:[i]});return{entity:i,documentID:v.documentIDs[0]??null}}
|
package/lib/api.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type {
|
|
1
|
+
import type { StoreProvider } from '@kubun/db';
|
|
2
2
|
export type SyncStateData = {
|
|
3
3
|
connectorName: string;
|
|
4
4
|
ownerDID: string;
|
|
@@ -25,5 +25,5 @@ export type ConnectorAPI = {
|
|
|
25
25
|
deleteCredential: (providerName: string, ownerDID: string) => Promise<void>;
|
|
26
26
|
};
|
|
27
27
|
export declare function createConnectorAPI(params: {
|
|
28
|
-
|
|
28
|
+
stores: StoreProvider;
|
|
29
29
|
}): ConnectorAPI;
|
package/lib/api.js
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
|
|
1
|
+
import{getConnectorStore as t}from"@kubun/store-connector";export function createConnectorAPI(e){return{async getSyncState(n,a){let r=await t(e.stores),s=await r.getSyncState(n,a);return null==s?null:{connectorName:n,ownerDID:a,checkpoint:null!=s.checkpoint?JSON.stringify(s.checkpoint):null,lastSyncedAt:s.lastSyncedAt,entityCount:s.entityCount,status:s.status,error:s.error??null}},async setSyncState(n,a,r){let s=await t(e.stores),c=r.lastSyncedAt??new Date().toISOString(),o=r.entityCount??0,l=r.checkpoint??null;await s.setSyncState(n,a,{checkpoint:null!=l?JSON.parse(l):null,lastSyncedAt:c,entityCount:o,status:r.status,error:r.error??void 0});let i=await s.getSyncState(n,a);if(null==i)throw Error("Failed to set sync state");return{connectorName:n,ownerDID:a,checkpoint:null!=i.checkpoint?JSON.stringify(i.checkpoint):null,lastSyncedAt:i.lastSyncedAt,entityCount:i.entityCount,status:i.status,error:i.error??null}},async deleteSyncState(n,a){let r=await t(e.stores);await r.deleteSyncState(n,a)},async listSyncStates(n){let a=await t(e.stores);return(await a.listSyncStatesForConnector(n)).map(t=>({connectorName:t.connectorName,ownerDID:t.ownerDID,checkpoint:null!=t.checkpoint?JSON.stringify(t.checkpoint):null,lastSyncedAt:t.lastSyncedAt,entityCount:t.entityCount,status:t.status,error:t.error??null}))},async getCredential(n,a){let r=await t(e.stores),s=await r.getCredential(n,a);return null==s?null:"string"==typeof s?s:JSON.stringify({accessToken:s.accessToken,refreshToken:s.refreshToken,expiresAt:s.expiresAt?.toISOString(),scopes:s.scopes,accountLabel:s.accountLabel,metadata:s.metadata})},async setCredential(n,a,r){let s=await t(e.stores);await s.setCredential(n,a,r)},async deleteCredential(n,a){let r=await t(e.stores);await r.deleteCredential(n,a)}}}
|
package/lib/index.js
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
import{fromEmitter as e}from"@enkaku/generator";import{
|
|
1
|
+
import{fromEmitter as e}from"@enkaku/generator";import{connectorStoreDefinition as t}from"@kubun/store-connector";import{executeAction as r}from"./action.js";import{createConnectorAPI as n}from"./api.js";import{DBCredentialProvider as o}from"./credential.js";import{ConnectorManager as c}from"./manager.js";import{OAuthService as s}from"./oauth.js";import{ConnectorRegistry as i}from"./registry.js";import{createConnectorSchemaExtension as a,toConnectorSyncEventResult as u}from"./schema.js";import{orchestrateSync as l}from"./sync/orchestrate.js";import{DBSyncStateStore as m}from"./sync/state.js";export{executeAction}from"./action.js";export{ConnectorManager}from"./manager.js";export{OAuthService}from"./oauth.js";export{ConnectorRegistry}from"./registry.js";export{createConnectorSchemaExtension,toConnectorSyncEventResult}from"./schema.js";export{SyncEngine}from"./sync/engine.js";export{orchestrateSync}from"./sync/orchestrate.js";export{EntityProcessor}from"./sync/processor.js";export{DBSyncStateStore}from"./sync/state.js";async function p(e,t,r,n,o){let c=t.get(o);if(null==c)throw Error(`Connector "${o}" not found`);let s=await e.getSyncState(o,n),i=!1,a=!1,u=null;if("device"!==c.auth.provider){let e=await r.get(c.auth.provider,n);if(null!=e){i=!0,null!=e.expiresAt&&(u=e.expiresAt.toISOString());let t=c.auth;a=null==t.writeScopes||t.writeScopes.every(t=>e.scopes.includes(t))}}return null==s?{name:o,status:"IDLE",authenticated:i,hasWriteAccess:a,authExpiresAt:u,lastSyncedAt:null,entityCount:null,error:null}:{name:o,status:"idle"===s.status?"IDLE":"syncing"===s.status?"SYNCING":"ERROR",authenticated:i,hasWriteAccess:a,authExpiresAt:u,lastSyncedAt:s.lastSyncedAt??null,entityCount:s.entityCount??null,error:s.error??null}}export function createConnectorPlugin(y){let f=y.providers??[];return d=>{d.db.register(t);let g=new i;for(let e of y.connectors)g.register(e);let S=n({stores:d.db}),D=new o({api:S,runtime:d.runtime,providers:f}),h=new m({api:S}),w=new c({registry:g,logger:d.getLogger("connector"),defaults:y.defaults}),j=new s({runtime:d.runtime,providers:f,registry:g,credentialProvider:D});return{name:"connector",schemaExtension:e=>a({registry:g,config:e}),api:S,createContextFactory:()=>(t,c)=>{let s=n({stores:c}),i=new o({api:s,runtime:d.runtime,providers:f});return{getState:e=>p(s,g,i,t.viewerDID,e),getStates:()=>Promise.all(g.list().map(e=>p(s,g,i,t.viewerDID,e))),startAuth:e=>j.startAuth(e),completeAuth:e=>j.completeAuth(e,t.viewerDID,i),triggerSync:async e=>g.has(e.connector)?(c.onCommit(()=>{l({connectorName:e.connector,full:e.full??!1,registry:g,syncEventEmitter:w,stateStore:h,credentialProvider:D,ownerDID:t.viewerDID,stores:d.db,boundary:y.defaults?.boundary}).catch(()=>{})}),{status:"STARTED",connectorName:e.connector}):{status:"ERROR",connectorName:e.connector},subscribeToSyncEvents:t=>{let r=null!=t?e=>e.connectorName===t:void 0,n=e(w.syncEvents,"sync",{filter:r});return async function*(){for await(let e of n)yield u(e)}()},executeAction:async e=>{try{let n=await r({connector:e.connector,action:e.action,model:e.model,input:e.input,sourceID:e.sourceID},{stores:c,registry:g,credentialProvider:i,ownerDID:t.viewerDID});return{documentID:n.documentID,entity:n.entity,error:null}}catch(t){let e=t.code??"API_ERROR";return{documentID:null,entity:null,error:{code:e,message:t instanceof Error?t.message:String(t),requiredScopes:t.requiredScopes}}}}}}}}}export{o as DBCredentialProvider};
|
package/lib/oauth.d.ts
CHANGED
|
@@ -12,5 +12,5 @@ export declare class OAuthService {
|
|
|
12
12
|
#private;
|
|
13
13
|
constructor(params: OAuthServiceParams);
|
|
14
14
|
startAuth(args: StartConnectorAuthInput): Promise<StartConnectorAuthOutput>;
|
|
15
|
-
completeAuth(args: CompleteConnectorAuthInput, viewerDID: string): Promise<CompleteConnectorAuthOutput>;
|
|
15
|
+
completeAuth(args: CompleteConnectorAuthInput, viewerDID: string, credentialProvider?: CredentialProvider): Promise<CompleteConnectorAuthOutput>;
|
|
16
16
|
}
|
package/lib/oauth.js
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
export class OAuthService{#e;#r;#t;#i;constructor(e){this.#e=e.credentialProvider,this.#r=e.runtime,this.#t=e.providers,this.#i=e.registry}async startAuth(e){let r=this.#t.find(r=>r.name===e.provider);if(null==r)throw Error(`OAuth provider "${e.provider}" not found`);if(null==r.clientID)throw Error(`OAuth provider "${e.provider}" is missing clientID`);let t=!0===e.requestWriteAccess,i=new Set(r.baseScopes??[]);for(let r of e.connectors??this.#i.list()){let o=this.#i.get(r);if(null!=o&&o.auth.provider===e.provider){let e=o.auth;if(t&&null!=e.writeScopes)for(let r of e.writeScopes)i.add(r);else for(let r of e.scopes)i.add(r)}}let o=JSON.stringify({nonce:this.#r.getRandomID(),provider:e.provider}),s=new URL(r.authorizationEndpoint);return s.searchParams.set("client_id",r.clientID),s.searchParams.set("redirect_uri",e.redirectURL),s.searchParams.set("response_type","code"),s.searchParams.set("scope",Array.from(i).join(" ")),s.searchParams.set("state",o),s.searchParams.set("access_type","offline"),s.searchParams.set("prompt","consent"),{url:s.toString(),state:o}}async completeAuth(e,r){let
|
|
1
|
+
export class OAuthService{#e;#r;#t;#i;constructor(e){this.#e=e.credentialProvider,this.#r=e.runtime,this.#t=e.providers,this.#i=e.registry}async startAuth(e){let r=this.#t.find(r=>r.name===e.provider);if(null==r)throw Error(`OAuth provider "${e.provider}" not found`);if(null==r.clientID)throw Error(`OAuth provider "${e.provider}" is missing clientID`);let t=!0===e.requestWriteAccess,i=new Set(r.baseScopes??[]);for(let r of e.connectors??this.#i.list()){let o=this.#i.get(r);if(null!=o&&o.auth.provider===e.provider){let e=o.auth;if(t&&null!=e.writeScopes)for(let r of e.writeScopes)i.add(r);else for(let r of e.scopes)i.add(r)}}let o=JSON.stringify({nonce:this.#r.getRandomID(),provider:e.provider}),s=new URL(r.authorizationEndpoint);return s.searchParams.set("client_id",r.clientID),s.searchParams.set("redirect_uri",e.redirectURL),s.searchParams.set("response_type","code"),s.searchParams.set("scope",Array.from(i).join(" ")),s.searchParams.set("state",o),s.searchParams.set("access_type","offline"),s.searchParams.set("prompt","consent"),{url:s.toString(),state:o}}async completeAuth(e,r,t){let i=this.#t.find(r=>r.name===e.provider);if(null==i)throw Error(`OAuth provider "${e.provider}" not found`);if(null==i.clientID||null==i.clientSecret)throw Error(`OAuth provider "${e.provider}" is missing clientID or clientSecret`);let o=await this.#r.fetch(i.tokenEndpoint,{method:"POST",headers:{"Content-Type":"application/x-www-form-urlencoded"},body:new URLSearchParams({grant_type:"authorization_code",code:e.code,redirect_uri:e.redirectURL,client_id:i.clientID,client_secret:i.clientSecret})});if(!o.ok)throw Error(`Token exchange failed: ${o.statusText}`);let s=await o.json(),n=s.scope?.split(" ")??[];return await (t??this.#e).set(e.provider,r,{accessToken:s.access_token,refreshToken:s.refresh_token,expiresAt:null!=s.expires_in?new Date(Date.now()+1e3*s.expires_in):void 0,scopes:n}),{providerName:e.provider,scopes:n}}}
|
package/lib/schema.d.ts
CHANGED
|
@@ -92,16 +92,24 @@ export declare function toConnectorSyncEventResult(event: ConnectorSyncEventPayl
|
|
|
92
92
|
/**
|
|
93
93
|
* Per-request context methods for the Connector query and mutation fields.
|
|
94
94
|
* Provided by the context factory when registry + credentials are configured.
|
|
95
|
+
*
|
|
96
|
+
* Fields land on the GraphQL resolver context under `ctx.connector` via the
|
|
97
|
+
* `declare module '@kubun/graphql'` augmentation below.
|
|
95
98
|
*/
|
|
96
99
|
export type ConnectorQueryContext = {
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
100
|
+
getState: (name: string) => Promise<ConnectorState>;
|
|
101
|
+
getStates: () => Promise<Array<ConnectorState>>;
|
|
102
|
+
startAuth: (args: StartConnectorAuthInput) => Promise<StartConnectorAuthOutput>;
|
|
103
|
+
completeAuth: (args: CompleteConnectorAuthInput) => Promise<CompleteConnectorAuthOutput>;
|
|
104
|
+
triggerSync: (args: SyncTriggerInput) => Promise<SyncTriggerOutput>;
|
|
105
|
+
subscribeToSyncEvents: (connector?: string | null) => AsyncIterable<ConnectorSyncEventResult>;
|
|
106
|
+
executeAction?: (args: ConnectorActionInput) => Promise<ConnectorActionResult>;
|
|
104
107
|
};
|
|
108
|
+
declare module '@kubun/graphql' {
|
|
109
|
+
interface PluginContextMap {
|
|
110
|
+
connector: ConnectorQueryContext;
|
|
111
|
+
}
|
|
112
|
+
}
|
|
105
113
|
export type ConnectorExtensionConfig = {
|
|
106
114
|
connectors?: Array<string>;
|
|
107
115
|
};
|
package/lib/schema.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import{PluginNodeID as t}from"@kubun/id";export function toConnectorSyncEventResult(t){return{type:"started"===t.type?"STARTED":"progress"===t.type?"PROGRESS":"completed"===t.type?"COMPLETED":"ERROR",connectorName:t.connectorName,entitiesProcessed:t.entitiesProcessed??null,entitiesFailed:t.entitiesFailed??null,totalProcessed:t.totalProcessed??null,totalFailed:t.totalFailed??null,duration:t.duration??null,error:t.error??null}}function e(t){let e=[];for(let n of Object.values(t.clusters))for(let t of Object.values(n.record)){let o=n.models[t];o?.name==null||e.includes(o.name)||e.push(o.name)}return e}export function createConnectorSchemaExtension(
|
|
1
|
+
import{PluginNodeID as t}from"@kubun/id";export function toConnectorSyncEventResult(t){return{type:"started"===t.type?"STARTED":"progress"===t.type?"PROGRESS":"completed"===t.type?"COMPLETED":"ERROR",connectorName:t.connectorName,entitiesProcessed:t.entitiesProcessed??null,entitiesFailed:t.entitiesFailed??null,totalProcessed:t.totalProcessed??null,totalFailed:t.totalFailed??null,duration:t.duration??null,error:t.error??null}}function e(t){if(null==t.connector)throw Error("connector plugin is not wired into this context");return t.connector}function n(t){let e=[];for(let n of Object.values(t.clusters))for(let t of Object.values(n.record)){let o=n.models[t];o?.name==null||e.includes(o.name)||e.push(o.name)}return e}export function createConnectorSchemaExtension(o){let r=o.registry,c=[];c.push(`
|
|
2
2
|
enum ConnectorSyncStatus {
|
|
3
3
|
IDLE
|
|
4
4
|
SYNCING
|
|
@@ -76,21 +76,21 @@ extend type Mutation {
|
|
|
76
76
|
extend type Subscription {
|
|
77
77
|
connectorSyncEvents(connector: String): ConnectorSyncEvent!
|
|
78
78
|
}
|
|
79
|
-
`);let
|
|
79
|
+
`);let i=[];if(null!=r){let t=new Set;for(let e of r.getAll()){if(null==e.actions||0===e.actions.length)continue;let o=e.actions.some(t=>"create"===t.name),r=e.actions.some(t=>"update"===t.name);if(o||r)for(let s of n(e).filter(t=>"ExternalEntity"!==t))t.has(s)||(t.add(s),c.push(`
|
|
80
80
|
type Connector${s}Payload {
|
|
81
81
|
documentID: String
|
|
82
82
|
entity: JSON
|
|
83
83
|
error: ConnectorActionError
|
|
84
84
|
}
|
|
85
|
-
`)),o&&(
|
|
85
|
+
`)),o&&(c.push(`
|
|
86
86
|
input ConnectorCreate${s}Input {
|
|
87
87
|
input: JSON!
|
|
88
88
|
}
|
|
89
|
-
`),
|
|
89
|
+
`),i.push(` connectorCreate${s}(input: ConnectorCreate${s}Input!): Connector${s}Payload!`)),r&&(c.push(`
|
|
90
90
|
input ConnectorUpdate${s}Input {
|
|
91
91
|
sourceID: String!
|
|
92
92
|
input: JSON!
|
|
93
93
|
}
|
|
94
|
-
`),
|
|
95
|
-
${
|
|
96
|
-
}`);let
|
|
94
|
+
`),i.push(` connectorUpdate${s}(input: ConnectorUpdate${s}Input!): Connector${s}Payload!`))}}i.length>0&&c.push(`extend type Mutation {
|
|
95
|
+
${i.join("\n")}
|
|
96
|
+
}`);let s=c.join("\n"),l={startConnectorAuth:(t,n,o)=>e(o).startAuth(n),completeConnectorAuth:(t,n,o)=>e(o).completeAuth(n),syncConnector:(t,n,o)=>e(o).triggerSync(n)};if(null!=r)for(let t of r.getAll()){if(null==t.actions||0===t.actions.length)continue;let o=t.actions.some(t=>"create"===t.name),r=t.actions.some(t=>"update"===t.name);if(o||r)for(let c of n(t).filter(t=>"ExternalEntity"!==t)){if(o){let n=t.name;l[`connectorCreate${c}`]=async(t,o,r)=>{let i=e(r);if(null==i.executeAction)throw Error("connector.executeAction is not available in this context");let s=o.input;return await i.executeAction({connector:n,action:"create",model:c,input:s.input})}}if(r){let n=t.name;l[`connectorUpdate${c}`]=async(t,o,r)=>{let i=e(r);if(null==i.executeAction)throw Error("connector.executeAction is not available in this context");let s=o.input;return await i.executeAction({connector:n,action:"update",model:c,input:s.input,sourceID:s.sourceID})}}}}return{sdl:s,resolvers:{queryFields:{connector:(t,n,o)=>e(o).getState(n.name),connectors:(t,n,o)=>e(o).getStates()},mutationFields:l,subscriptionFields:{connectorSyncEvents:{resolve:t=>t,subscribe:(t,n,o)=>e(o).subscribeToSyncEvents(n.connector)}},typeFields:{Connector:{id:e=>t.create("connector","Connector",e.name).toString()}},nodeResolvers:{Connector:{resolve:(t,n)=>e(n).getState(t)}}}}}
|
package/lib/sync/engine.d.ts
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
import type { DataProvider, SyncBoundary, SyncEvent, SyncStateStore } from '@kubun/connector';
|
|
2
|
-
import type {
|
|
2
|
+
import type { StoreProvider } from '@kubun/db';
|
|
3
3
|
import type { ClustersRecord } from '@kubun/protocol';
|
|
4
4
|
export type SyncEngineParams = {
|
|
5
|
-
|
|
5
|
+
stores: StoreProvider;
|
|
6
6
|
stateStore: SyncStateStore;
|
|
7
7
|
connectorName: string;
|
|
8
8
|
clusters: ClustersRecord;
|
package/lib/sync/engine.js
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
import{EventEmitter as t}from"@enkaku/event";import{EntityProcessor as e}from"./processor.js";export class SyncEngine{#t;#e;#n;#s;#o;#c=new Map;#r=new t;constructor(t){for(let e of(this.#t=t.
|
|
1
|
+
import{EventEmitter as t}from"@enkaku/event";import{EntityProcessor as e}from"./processor.js";export class SyncEngine{#t;#e;#n;#s;#o;#c=new Map;#r=new t;constructor(t){for(let e of(this.#t=t.stores,this.#e=t.stateStore,this.#n=t.connectorName,this.#s=t.clusters,this.#o=new Map,Object.values(t.clusters)))for(let[t,n]of Object.entries(e.record))this.#o.set(t,e.models[n])}on(t,e){return this.#r.on(t,e)}#i(t,n){let s=this.#c.get(t);if(null!=s)return s;let o=this.#o.get(t);if(null==o)throw Error(`Unknown model ID "${t}" in connector "${this.#n}"`);let c=new e({stores:this.#t,modelID:t,ownerDID:n,connectorName:this.#n,edgeFields:o.fieldsMeta,clusters:this.#s});return this.#c.set(t,c),c}async run(t){let{provider:e,ownerDID:n,boundary:s,full:o,signal:c}=t,r=Date.now(),i=await this.#e.get(this.#n,n),a=!o&&i?.checkpoint!=null,l=a?"incremental":"initial";await this.#e.set(this.#n,n,{checkpoint:i?.checkpoint??null,lastSyncedAt:i?.lastSyncedAt??new Date().toISOString(),entityCount:i?.entityCount??0,status:"syncing"});let h=0,y=0,m=0,u=i?.checkpoint??null;try{let t={boundary:s,checkpoint:a?i?.checkpoint:void 0,signal:c};for await(let s of a?e.fetchChanges({...t,lastSyncedAt:i?.lastSyncedAt?new Date(i.lastSyncedAt):void 0}):e.fetchAll(t)){m++;let t=this.#i(s.modelID,n),e=await t.processBatch(s);h+=e.created+e.updated+e.deleted,y+=e.failed,null!=s.checkpoint&&(u=s.checkpoint);let o={type:"sync:progress",connectorName:this.#n,phase:l,entitiesProcessed:h,entitiesFailed:y,currentBatch:m,checkpoint:u};await this.#r.emit("sync",o),await this.#e.set(this.#n,n,{checkpoint:u,lastSyncedAt:new Date().toISOString(),entityCount:(i?.entityCount??0)+h,status:"syncing"})}let o=Date.now()-r;await this.#e.set(this.#n,n,{checkpoint:u,lastSyncedAt:new Date().toISOString(),entityCount:(i?.entityCount??0)+h,status:"idle"});let d={type:"sync:complete",connectorName:this.#n,totalProcessed:h,totalFailed:y,duration:o};await this.#r.emit("sync",d)}catch(s){let t=s instanceof Error?s.message:String(s);await this.#e.set(this.#n,n,{checkpoint:u,lastSyncedAt:i?.lastSyncedAt??new Date().toISOString(),entityCount:i?.entityCount??0,status:"error",error:t});let e={type:"sync:error",connectorName:this.#n,error:t};await this.#r.emit("sync",e)}}}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { ConnectorSyncEventPayload, CredentialProvider, SyncBoundary, SyncStateStore } from '@kubun/connector';
|
|
2
|
-
import type {
|
|
2
|
+
import type { StoreProvider } from '@kubun/db';
|
|
3
3
|
import type { ConnectorRegistry } from '../registry.js';
|
|
4
4
|
/**
|
|
5
5
|
* Minimal interface for the sync event emitter dependency.
|
|
@@ -17,7 +17,7 @@ export type OrchestrateSyncParams = {
|
|
|
17
17
|
stateStore: SyncStateStore;
|
|
18
18
|
credentialProvider: CredentialProvider;
|
|
19
19
|
ownerDID: string;
|
|
20
|
-
|
|
20
|
+
stores: StoreProvider;
|
|
21
21
|
boundary?: SyncBoundary;
|
|
22
22
|
};
|
|
23
23
|
export type OrchestrateSyncResult = {
|
package/lib/sync/orchestrate.js
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
import{SyncEngine as e}from"./engine.js";export async function orchestrateSync(t){let{connectorName:r,full:n,registry:s,syncEventEmitter:a,stateStore:i,credentialProvider:o,ownerDID:c,
|
|
1
|
+
import{SyncEngine as e}from"./engine.js";export async function orchestrateSync(t){let{connectorName:r,full:n,registry:s,syncEventEmitter:a,stateStore:i,credentialProvider:o,ownerDID:c,stores:l}=t,u=s.get(r);if(null==u)return{status:"error"};let y=await i.get(r,c);if(y?.status==="syncing")return{status:"already_syncing"};let d="device"!==u.auth.provider?u.auth.provider:null,p=d?await o.get(d,c):null;if(null!=d&&null==p||null==u.serverProvider)return{status:"error"};let v=t.boundary??{maxAge:"P90D"},g=u.serverProvider({credential:p??{accessToken:"",scopes:[]},boundary:v}),m=new e({stores:l,stateStore:i,connectorName:r,clusters:u.clusters});return await a.emitSyncEvent({type:"started",connectorName:r}),m.on("sync",e=>{"sync:progress"===e.type?a.emitSyncEvent({type:"progress",connectorName:r,entitiesProcessed:e.entitiesProcessed,entitiesFailed:e.entitiesFailed}):"sync:complete"===e.type?a.emitSyncEvent({type:"completed",connectorName:r,totalProcessed:e.totalProcessed,totalFailed:e.totalFailed,duration:e.duration}):"sync:error"===e.type&&a.emitSyncEvent({type:"error",connectorName:r,error:e.error})}),m.run({provider:g,ownerDID:c,boundary:v,full:n}).catch(async e=>{await a.emitSyncEvent({type:"error",connectorName:r,error:e instanceof Error?e.message:String(e)})}),{status:"started"}}
|
package/lib/sync/processor.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { EntityBatch } from '@kubun/connector';
|
|
2
|
-
import type {
|
|
2
|
+
import type { StoreProvider } from '@kubun/db';
|
|
3
3
|
import type { ClustersRecord, DocumentFieldsMeta } from '@kubun/protocol';
|
|
4
4
|
export type ProcessBatchResult = {
|
|
5
5
|
created: number;
|
|
@@ -9,7 +9,7 @@ export type ProcessBatchResult = {
|
|
|
9
9
|
documentIDs: Array<string>;
|
|
10
10
|
};
|
|
11
11
|
export type EntityProcessorParams = {
|
|
12
|
-
|
|
12
|
+
stores: StoreProvider;
|
|
13
13
|
modelID: string;
|
|
14
14
|
ownerDID: string;
|
|
15
15
|
connectorName: string;
|
package/lib/sync/processor.js
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
import{DocumentID as e,DocumentModelID as t}from"@kubun/id";export class EntityProcessor{#e;#t;#r;#o;#
|
|
1
|
+
import{DocumentID as e,DocumentModelID as t}from"@kubun/id";import{getGraphStore as r}from"@kubun/store-graph";export class EntityProcessor{#e;#t=null;#r;#o;#s;#n;#c;constructor(e){this.#e=e.stores,this.#r=t.fromString(e.modelID),this.#o=e.ownerDID,this.#s=e.connectorName,this.#n=e.edgeFields??{},this.#c=e.clusters}#i(t,r){let o=new TextEncoder().encode(`${t}:${r}`);return e.create(this.#r,this.#o,o)}#l(e){let r,o=this.#c;if(null==o)return;for(let t of Object.values(o))if(null!=t.record[e])return e;try{r=t.fromString(e)}catch{return}if(!r.isLocal)return;let s=r.index;for(let e of Object.values(o))for(let[t,r]of Object.entries(e.record))if(r===s)return t}#u(r){if(null==this.#c||0===Object.keys(this.#n).length)return r;let o={...r};for(let[s,n]of Object.entries(this.#n)){if("document"!==n.type||null==n.model)continue;let c=r[s];if(null==c||"object"!=typeof c)continue;let i=c.sourceService,l=c.sourceID;if("string"!=typeof i||"string"!=typeof l)continue;let u=this.#l(n.model);if(null==u)continue;let d=new TextEncoder().encode(`${i}:${l}`),a=e.create(t.fromString(u),this.#o,d);o[s]=a.toString()}return o}async processBatch(e){this.#t??=r(this.#e);let t=await this.#t,o=0,s=0,n=0,c=0,i=[];for(let r of e.entities)try{let e=this.#i(r.sourceService,r.sourceID),n=await t.getDocument(e),c={...r,lastSyncedAt:new Date().toISOString()},l=this.#u(c);if(null==n){let s=new TextEncoder().encode(`${r.sourceService}:${r.sourceID}`);await t.createDocument({id:e,owner:this.#o,unique:s,data:l}),o++}else await t.saveDocument({id:e,existing:n,data:l}),s++;i.push(e.toString())}catch{c++}if(e.deleted)for(let r of e.deleted)try{let e=this.#i(r.sourceService,r.sourceID),o=await t.getDocument(e);null!=o&&(await t.saveDocument({id:e,existing:o,data:{sourceService:r.sourceService,sourceID:r.sourceID,lastSyncedAt:new Date().toISOString(),_deleted:!0}}),n++)}catch{c++}return{created:o,updated:s,deleted:n,failed:c,documentIDs:i}}}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@kubun/plugin-connector",
|
|
3
|
-
"version": "0.8.
|
|
3
|
+
"version": "0.8.2",
|
|
4
4
|
"license": "see LICENSE.md",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "lib/index.js",
|
|
@@ -14,7 +14,7 @@
|
|
|
14
14
|
],
|
|
15
15
|
"sideEffects": false,
|
|
16
16
|
"dependencies": {
|
|
17
|
-
"@enkaku/async": "^0.14.
|
|
17
|
+
"@enkaku/async": "^0.14.1",
|
|
18
18
|
"@enkaku/event": "^0.14.1",
|
|
19
19
|
"@enkaku/generator": "^0.14.0",
|
|
20
20
|
"@enkaku/runtime": "^0.14.0",
|
|
@@ -22,17 +22,19 @@
|
|
|
22
22
|
"graphql-scalars": "^1.25.0",
|
|
23
23
|
"kysely": "^0.28.16",
|
|
24
24
|
"@kubun/connector": "^0.8.0",
|
|
25
|
-
"@kubun/
|
|
26
|
-
"@kubun/
|
|
27
|
-
"@kubun/
|
|
28
|
-
"@kubun/
|
|
25
|
+
"@kubun/engine": "^0.8.3",
|
|
26
|
+
"@kubun/db": "^0.8.1",
|
|
27
|
+
"@kubun/store-graph": "^0.8.1",
|
|
28
|
+
"@kubun/store-connector": "^0.8.0",
|
|
29
|
+
"@kubun/graphql": "^0.8.2",
|
|
29
30
|
"@kubun/logger": "^0.8.0",
|
|
30
|
-
"@kubun/protocol": "^0.8.
|
|
31
|
+
"@kubun/protocol": "^0.8.1",
|
|
32
|
+
"@kubun/id": "^0.8.0"
|
|
31
33
|
},
|
|
32
34
|
"devDependencies": {
|
|
33
35
|
"@testcontainers/postgresql": "^11.14.0",
|
|
34
36
|
"get-port": "^7.2.0",
|
|
35
|
-
"@kubun/db-postgres": "^0.8.
|
|
37
|
+
"@kubun/db-postgres": "^0.8.1",
|
|
36
38
|
"@kubun/test-utils": "^0.8.0"
|
|
37
39
|
},
|
|
38
40
|
"scripts": {
|
package/lib/migration.d.ts
DELETED
package/lib/migration.js
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
export function createConnectorMigrations(t){let e=t.types;return{"001_create_connector_tables":{async up(t){await t.schema.createTable("kubun_connector_sync_state").ifNotExists().addColumn("connector_name",e.text,t=>t.notNull()).addColumn("owner_did",e.text,t=>t.notNull()).addColumn("checkpoint",e.text).addColumn("last_synced_at",e.text,t=>t.notNull()).addColumn("entity_count","integer",t=>t.notNull().defaultTo(0)).addColumn("status",e.text,t=>t.notNull()).addColumn("error",e.text).addPrimaryKeyConstraint("kubun_connector_sync_state_pkey",["connector_name","owner_did"]).execute(),await t.schema.createTable("kubun_connector_credentials").ifNotExists().addColumn("provider_name",e.text,t=>t.notNull()).addColumn("owner_did",e.text,t=>t.notNull()).addColumn("credential",e.text,t=>t.notNull()).addPrimaryKeyConstraint("kubun_connector_credentials_pkey",["provider_name","owner_did"]).execute()},async down(t){await t.schema.dropTable("kubun_connector_credentials").ifExists().execute(),await t.schema.dropTable("kubun_connector_sync_state").ifExists().execute()}}}}
|