@project-ajax/sdk 0.0.56 → 0.0.59

package/dist/capabilities/sync.d.ts

@@ -19,11 +19,33 @@ export type SyncedObject<PK extends string, S extends PropertySchema<PK>> = {
      */
     pageContentMarkdown?: string;
 };
+/**
+ * Result returned from the sync execute function.
+ */
+export type SyncExecutionResult<PK extends string, Context = unknown> = {
+    /**
+     * The batch of objects fetched in this execution.
+     */
+    objects: SyncedObject<PK, PropertySchema<PK>>[];
+    /**
+     * Indicates whether the sync is complete.
+     * - `true`: No more data to fetch, sync is complete
+     * - `false`: More data available, will trigger another execution with nextContext
+     */
+    done: boolean;
+    /**
+     * Optional context data to pass to the next execution.
+     * Required if `done` is `false`, ignored if `done` is `true`.
+     * This can be any type of data (cursor, page number, timestamp, etc.).
+     * The same data will be provided in the context parameter of the next execution.
+     */
+    nextContext?: Context;
+};
 /**
  * A configuration object that enables synchronization between a data
  * source and a third-party source.
  */
-export type SyncConfiguration<PK extends string, S extends Schema<PK>> = {
+export type SyncConfiguration<PK extends string, S extends Schema<PK>, Context = unknown> = {
     /**
      * The property of the data source that maps to a "primary key" in the
      * third-party data. This is used to match existing pages to
@@ -34,18 +56,38 @@ export type SyncConfiguration<PK extends string, S extends Schema<PK>> = {
      * The schema defining the structure of properties in the collection.
      */
     schema: S;
+    /**
+     * Whether to delete pages from the collection that are not returned
+     * from any of the sync executions (mark-and-sweep deletion).
+     *
+     * - `true`: Pages not seen in any execution batch will be deleted
+     * - `false` (default): Only upsert pages, never delete
+     *
+     * @default false
+     */
+    deleteUnreturnedPages?: boolean;
     /**
      * A function that fetches the data to sync from the third-party service.
      *
-     * For now, this function must return all of the data to be synced each time
-     * it's called. The runtime will handle diffing this against the data source
-     * and creating, updating, and deleting pages as necessary.
+     * This function can return all data at once, or implement pagination by:
+     * 1. Returning a batch of objects with `done: false` and a `nextContext`
+     * 2. The runtime will call execute again with that context data
+     * 3. Continue until `done: true` is returned
+     *
+     * The runtime will handle diffing against the data source and creating,
+     * updating, and deleting pages as necessary.
+     *
+     * @param context - Optional context data from previous execution (undefined on first call)
+     * @returns A result containing objects, done status, and optional nextContext
      */
-    execute: () => Promise<SyncedObject<PK, PropertySchema<PK>>[]>;
+    execute: (context?: Context) => Promise<SyncExecutionResult<PK, Context>>;
 };
-export type SyncHandlerResult<PK extends string> = {
+export type SyncHandlerResult<PK extends string, Context = unknown> = {
     primaryKeyProperty: PK;
     objects: SyncedObject<PK, PropertySchema<PK>>[];
+    done: boolean;
+    nextContext?: Context;
+    deleteUnreturnedPages?: boolean;
 };
 /**
  * Creates a special handler for syncing third-party data to a collection.
@@ -54,15 +96,17 @@ export type SyncHandlerResult<PK extends string> = {
  * @returns A handler function that executes the sync function, and passes data
  * needed to complete the sync back to the platform.
  */
-export declare function sync<PK extends string, S extends Schema<PK>>(syncConfiguration: SyncConfiguration<PK, S>): {
+export declare function sync<PK extends string, S extends Schema<PK>, Context = unknown>(syncConfiguration: SyncConfiguration<PK, S, Context>): {
     _tag: string;
     config: {
+        primaryKeyProperty: PK;
         schema: S;
+        deleteUnreturnedPages: boolean | undefined;
     };
-    handler(): Promise<{
-        schema: S;
-        primaryKeyProperty: PK;
+    handler(context?: Context): Promise<{
         objects: SyncedObject<PK, PropertySchema<PK>>[];
+        done: boolean;
+        nextContext: Context | undefined;
     }>;
 };
 //# sourceMappingURL=sync.d.ts.map

package/dist/capabilities/sync.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"sync.d.ts","sourceRoot":"","sources":["../../src/capabilities/sync.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,EAAE,MAAM,cAAc,CAAC;AAC3D,OAAO,KAAK,EAAE,IAAI,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAEnD;;GAEG;AACH,MAAM,MAAM,YAAY,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC,SAAS,cAAc,CAAC,EAAE,CAAC,IAAI;IAC3E,GAAG,EAAE,MAAM,CAAC;IACZ,UAAU,EAAE;SAAG,QAAQ,IAAI,MAAM,CAAC,GAAG,SAAS;KAAE,CAAC;IACjD;;;OAGG;IACH,IAAI,CAAC,EAAE,IAAI,CAAC;IACZ;;;OAGG;IACH,mBAAmB,CAAC,EAAE,MAAM,CAAC;CAC7B,CAAC;AAEF;;;GAGG;AACH,MAAM,MAAM,iBAAiB,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC,SAAS,MAAM,CAAC,EAAE,CAAC,IAAI;IACxE;;;;OAIG;IACH,kBAAkB,EAAE,EAAE,CAAC;IAEvB;;OAEG;IACH,MAAM,EAAE,CAAC,CAAC;IAEV;;;;;;OAMG;IACH,OAAO,EAAE,MAAM,OAAO,CAAC,YAAY,CAAC,EAAE,EAAE,cAAc,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC;CAC/D,CAAC;AAEF,MAAM,MAAM,iBAAiB,CAAC,EAAE,SAAS,MAAM,IAAI;IAClD,kBAAkB,EAAE,EAAE,CAAC;IACvB,OAAO,EAAE,YAAY,CAAC,EAAE,EAAE,cAAc,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC;CAChD,CAAC;AAEF;;;;;;GAMG;AACH,wBAAgB,IAAI,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC,SAAS,MAAM,CAAC,EAAE,CAAC,EAC3D,iBAAiB,EAAE,iBAAiB,CAAC,EAAE,EAAE,CAAC,CAAC;;;;;;;;;;EAuB3C"}
+{"version":3,"file":"sync.d.ts","sourceRoot":"","sources":["../../src/capabilities/sync.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,EAAE,MAAM,cAAc,CAAC;AAC3D,OAAO,KAAK,EAAE,IAAI,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAEnD;;GAEG;AACH,MAAM,MAAM,YAAY,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC,SAAS,cAAc,CAAC,EAAE,CAAC,IAAI;IAC3E,GAAG,EAAE,MAAM,CAAC;IACZ,UAAU,EAAE;SAAG,QAAQ,IAAI,MAAM,CAAC,GAAG,SAAS;KAAE,CAAC;IACjD;;;OAGG;IACH,IAAI,CAAC,EAAE,IAAI,CAAC;IACZ;;;OAGG;IACH,mBAAmB,CAAC,EAAE,MAAM,CAAC;CAC7B,CAAC;AAEF;;GAEG;AACH,MAAM,MAAM,mBAAmB,CAAC,EAAE,SAAS,MAAM,EAAE,OAAO,GAAG,OAAO,IAAI;IACvE;;OAEG;IACH,OAAO,EAAE,YAAY,CAAC,EAAE,EAAE,cAAc,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC;IAEhD;;;;OAIG;IACH,IAAI,EAAE,OAAO,CAAC;IAEd;;;;;OAKG;IACH,WAAW,CAAC,EAAE,OAAO,CAAC;CACtB,CAAC;AAEF;;;GAGG;AACH,MAAM,MAAM,iBAAiB,CAC5B,EAAE,SAAS,MAAM,EACjB,CAAC,SAAS,MAAM,CAAC,EAAE,CAAC,EACpB,OAAO,GAAG,OAAO,IACd;IACH;;;;OAIG;IACH,kBAAkB,EAAE,EAAE,CAAC;IAEvB;;OAEG;IACH,MAAM,EAAE,CAAC,CAAC;IAEV;;;;;;;;OAQG;IACH,qBAAqB,CAAC,EAAE,OAAO,CAAC;IAEhC;;;;;;;;;;;;;OAaG;IACH,OAAO,EAAE,CAAC,OAAO,CAAC,EAAE,OAAO,KAAK,OAAO,CAAC,mBAAmB,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC,CAAC;CAC1E,CAAC;AAEF,MAAM,MAAM,iBAAiB,CAAC,EAAE,SAAS,MAAM,EAAE,OAAO,GAAG,OAAO,IAAI;IACrE,kBAAkB,EAAE,EAAE,CAAC;IACvB,OAAO,EAAE,YAAY,CAAC,EAAE,EAAE,cAAc,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC;IAChD,IAAI,EAAE,OAAO,CAAC;IACd,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,qBAAqB,CAAC,EAAE,OAAO,CAAC;CAChC,CAAC;AAEF;;;;;;GAMG;AACH,wBAAgB,IAAI,CACnB,EAAE,SAAS,MAAM,EACjB,CAAC,SAAS,MAAM,CAAC,EAAE,CAAC,EACpB,OAAO,GAAG,OAAO,EAChB,iBAAiB,EAAE,iBAAiB,CAAC,EAAE,EAAE,CAAC,EAAE,OAAO,CAAC;;;;;;;sBAQ5B,OAAO;;;;;EAkBhC"}

package/dist/capabilities/sync.js

@@ -3,16 +3,18 @@ function sync(syncConfiguration) {
   return {
     _tag: "sync",
     config: {
-      schema: syncConfiguration.schema
+      primaryKeyProperty: syncConfiguration.primaryKeyProperty,
+      schema: syncConfiguration.schema,
+      deleteUnreturnedPages: syncConfiguration.deleteUnreturnedPages
     },
-    async handler() {
-      const objects = await syncConfiguration.execute().catch((err) => {
+    async handler(context) {
+      const executionResult = await syncConfiguration.execute(context).catch((err) => {
         throw new ExecutionError(err);
       });
       const result = {
-        schema: syncConfiguration.schema,
-        primaryKeyProperty: syncConfiguration.primaryKeyProperty,
-        objects
+        objects: executionResult.objects,
+        done: executionResult.done,
+        nextContext: executionResult.nextContext
       };
       process.stdout.write(`
 <output>${JSON.stringify(result)}</output>

package/dist/cli/commands/auth.impl.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"auth.impl.d.ts","sourceRoot":"","sources":["../../../src/cli/commands/auth.impl.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAI/C,eAAO,MAAM,KAAK,+GAiChB,CAAC;AAEH,eAAO,MAAM,IAAI,mFAEf,CAAC;AAEH,eAAO,MAAM,MAAM,mFAIjB,CAAC"}
+{"version":3,"file":"auth.impl.d.ts","sourceRoot":"","sources":["../../../src/cli/commands/auth.impl.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAI/C,eAAO,MAAM,KAAK,+GAwChB,CAAC;AAEH,eAAO,MAAM,IAAI,mFAEf,CAAC;AAEH,eAAO,MAAM,MAAM,mFAIjB,CAAC"}

package/dist/cli/commands/auth.impl.js

@@ -19,11 +19,15 @@ ${url}
     }
     return;
   }
-  await this.config.update({
-    environment,
+  const update = {
     token,
     workerId: null
-  });
+  };
+  console.log("environment", environment);
+  if (environment !== "prod") {
+    update.environment = environment;
+  }
+  await this.config.update(update);
   this.io.writeErr("Successfully logged in!");
 });
 const show = buildHandler(function() {

package/dist/cli/config.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../../src/cli/config.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAE9C,eAAO,MAAM,YAAY,8CAA+C,CAAC;AACzE,MAAM,MAAM,WAAW,GAAG,CAAC,OAAO,YAAY,CAAC,CAAC,MAAM,CAAC,CAAC;AAExD,MAAM,WAAW,SAAS;IACzB,WAAW,EAAE,WAAW,CAAC;IACzB,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;CACxB;AAED,wBAAgB,gBAAgB,CAAC,IAAI,EAAE,MAAM,GAAG,WAAW,CAM1D;AAED,qBAAa,gBAAiB,SAAQ,KAAK;gBAEzC,OAAO,GAAE,MAA6D;CAKvE;AAED;;;;;;;;;;;GAWG;AACH,qBAAa,MAAM;;gBAIN,IAAI,EAAE;QACjB,SAAS,EAAE,SAAS,CAAC;QACrB,cAAc,EAAE,MAAM,CAAC;KACvB;IAOD,IAAI,OAAO,WAEV;IAED,IAAI,KAAK,kBAER;IAED,IAAI,WAAW,yCAEd;IAED,IAAI,QAAQ,kBAEX;IAED,IAAI,SAAS,IAAI;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAQlE;IAED;;;;;;;;;OASG;IACG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,SAAS,CAAC;WAc1B,IAAI,CAAC,IAAI,EAAE;QACvB,cAAc,EAAE,MAAM,CAAC;QACvB,UAAU,EAAE,MAAM,CAAC,UAAU,CAAC;QAC9B,KAAK,EAAE,WAAW,CAAC;KACnB;CA+FD;AAQD,wBAAgB,uBAAuB,CAAC,KAAK,EAAE,MAAM,GAAG;IACvD,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;CACf,CAoCA"}
+{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../../src/cli/config.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAE9C,eAAO,MAAM,YAAY,8CAA+C,CAAC;AACzE,MAAM,MAAM,WAAW,GAAG,CAAC,OAAO,YAAY,CAAC,CAAC,MAAM,CAAC,CAAC;AAExD,MAAM,WAAW,SAAS;IACzB,WAAW,EAAE,WAAW,CAAC;IACzB,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;CACxB;AAED,wBAAgB,gBAAgB,CAAC,IAAI,EAAE,MAAM,GAAG,WAAW,CAM1D;AAED,qBAAa,gBAAiB,SAAQ,KAAK;gBAEzC,OAAO,GAAE,MAA6D;CAKvE;AAED;;;;;;;;;;;GAWG;AACH,qBAAa,MAAM;;gBAIN,IAAI,EAAE;QACjB,SAAS,EAAE,SAAS,CAAC;QACrB,cAAc,EAAE,MAAM,CAAC;KACvB;IAOD,IAAI,OAAO,WAEV;IAED,IAAI,KAAK,kBAER;IAED,IAAI,WAAW,yCAEd;IAED,IAAI,QAAQ,kBAEX;IAED,IAAI,SAAS,IAAI;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAQlE;IAED;;;;;;;;;OASG;IACG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,SAAS,CAAC;WAc1B,IAAI,CAAC,IAAI,EAAE;QACvB,cAAc,EAAE,MAAM,CAAC;QACvB,UAAU,EAAE,MAAM,CAAC,UAAU,CAAC;QAC9B,KAAK,EAAE,WAAW,CAAC;KACnB;CA4FD;AAQD,wBAAgB,uBAAuB,CAAC,KAAK,EAAE,MAAM,GAAG;IACvD,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;CACf,CAoCA"}

package/dist/cli/config.js

@@ -92,9 +92,8 @@ class Config {
     if (opts.flags["worker-id"]) {
       partialConfig.workerId = opts.flags["worker-id"];
     }
-    partialConfig.baseUrl ??= baseUrlForEnvironment(
-      partialConfig.environment ?? "prod"
-    );
+    partialConfig.environment ??= "prod";
+    partialConfig.baseUrl ??= baseUrlForEnvironment(partialConfig.environment);
     const environment = partialConfig.environment;
     if (!environment) {
       throw new Error("Environment is required");
@@ -134,9 +133,7 @@ class Config {
       if (isENOENT(error)) {
         configFile = {
           token: null,
-          workerId: null,
-          environment: "prod",
-          baseUrl: baseUrlForEnvironment("prod")
+          workerId: null
         };
       } else {
         throw error;

package/dist/cli/io.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"io.d.ts","sourceRoot":"","sources":["../../src/cli/io.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,OAAO,MAAM,mBAAmB,CAAC;AAC7C,OAAO,EAAe,KAAK,SAAS,EAAkB,MAAM,mBAAmB,CAAC;AAEhF,MAAM,WAAW,aAAa;IAC7B,YAAY,EAAE,OAAO,CAAC;CACtB;AAED,MAAM,WAAW,YAAY;IAC5B,OAAO,EAAE,SAAS,EAAE,GAAG,SAAS,EAAE,EAAE,CAAC;IACrC,IAAI,EAAE,SAAS,EAAE,EAAE,CAAC;IACpB,KAAK,EAAE,OAAO,CAAC;CACf;AAED,KAAK,UAAU,CAAC,CAAC,IAAI,CAAC,GAAG;IACxB,KAAK,EAAE,MAAM,CAAC;CACd,CAAC;AAEF;;GAEG;AACH,qBAAa,EAAE;;IACd,YAAY,EAAE,OAAO,CAAC;gBAEV,OAAO,EAAE,aAAa;IAIlC;;;;OAIG;IACH,KAAK,CAAC,GAAG,IAAI,EAAE,UAAU,CAAC,OAAO,OAAO,CAAC,GAAG,CAAC;IAW7C;;;;OAIG;IACH,QAAQ,CAAC,GAAG,IAAI,EAAE,UAAU,CAAC,OAAO,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC;IAIzD;;;;OAIG;IACH,QAAQ,CAAC,GAAG,IAAI,EAAE,UAAU,CAAC,OAAO,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC;IAIzD;;;;OAIG;IACH,aAAa,CAAC,WAAW,EAAE,YAAY;IAKvC;;;;OAIG;IACH,aAAa,CAAC,WAAW,EAAE,YAAY;IAKvC,OAAO,CAAC,MAAM,EAAE,UAAU,CAAC,UAAU,CAAC,OAAO,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;IASjE,KAAK,CAAC,MAAM,EAAE,UAAU,CAAC,UAAU,CAAC,OAAO,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;CAkC7D"}
+{"version":3,"file":"io.d.ts","sourceRoot":"","sources":["../../src/cli/io.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,OAAO,MAAM,mBAAmB,CAAC;AAE7C,OAAO,EAAe,KAAK,SAAS,EAAkB,MAAM,mBAAmB,CAAC;AAEhF,MAAM,WAAW,aAAa;IAC7B,YAAY,EAAE,OAAO,CAAC;CACtB;AAED,MAAM,WAAW,YAAY;IAC5B,OAAO,EAAE,SAAS,EAAE,GAAG,SAAS,EAAE,EAAE,CAAC;IACrC,IAAI,EAAE,SAAS,EAAE,EAAE,CAAC;IACpB,KAAK,EAAE,OAAO,CAAC;CACf;AAED,KAAK,UAAU,CAAC,CAAC,IAAI,CAAC,GAAG;IACxB,KAAK,EAAE,MAAM,CAAC;CACd,CAAC;AAEF;;GAEG;AACH,qBAAa,EAAE;;IACd,YAAY,EAAE,OAAO,CAAC;gBAEV,OAAO,EAAE,aAAa;IAIlC;;;;OAIG;IACH,KAAK,CAAC,GAAG,IAAI,EAAE,UAAU,CAAC,OAAO,OAAO,CAAC,GAAG,CAAC;IAW7C;;;;OAIG;IACH,QAAQ,CAAC,GAAG,IAAI,EAAE,UAAU,CAAC,OAAO,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC;IAIzD;;;;OAIG;IACH,QAAQ,CAAC,GAAG,IAAI,EAAE,UAAU,CAAC,OAAO,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC;IAIzD;;;;OAIG;IACH,aAAa,CAAC,WAAW,EAAE,YAAY;IAKvC;;;;OAIG;IACH,aAAa,CAAC,WAAW,EAAE,YAAY;IAKvC,OAAO,CAAC,MAAM,EAAE,UAAU,CAAC,UAAU,CAAC,OAAO,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;IAKjE,KAAK,CAAC,MAAM,EAAE,UAAU,CAAC,UAAU,CAAC,OAAO,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;CAuC7D"}

package/dist/cli/io.js

@@ -54,18 +54,19 @@ class IO {
     this.writeErr(content);
   }
   confirm(config) {
-    if (!process.stdin.isTTY) {
-      this.writeErr(config.noTTY);
-      process.exit(1);
-    }
+    this.#ensureTTY(config.noTTY);
     return prompts.confirm(config).catch(this.#handlePromptExit.bind(this));
   }
   input(config) {
+    this.#ensureTTY(config.noTTY);
+    return prompts.input(config).catch(this.#handlePromptExit.bind(this));
+  }
+  #ensureTTY(noTTY) {
     if (!process.stdin.isTTY) {
-      this.writeErr(config.noTTY);
+      this.writeErr(noTTY);
       process.exit(1);
     }
-    return prompts.input(config).catch(this.#handlePromptExit.bind(this));
+    return true;
   }
   #handlePromptExit(err) {
     if (err instanceof Error && err.name === "ExitPromptError") {

package/dist/index.d.ts

@@ -1,5 +1,6 @@
 export { emojiIcon, notionIcon } from "./builder.js";
 export { slashCommand } from "./capabilities/slashCommand.js";
+export type { SyncConfiguration, SyncExecutionResult, SyncedObject, } from "./capabilities/sync.js";
 export { sync } from "./capabilities/sync.js";
 export { tool } from "./capabilities/tool.js";
 export type { Icon, NoticonColor, NoticonName } from "./types.js";

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AACrD,OAAO,EAAE,YAAY,EAAE,MAAM,gCAAgC,CAAC;AAC9D,OAAO,EAAE,IAAI,EAAE,MAAM,wBAAwB,CAAC;AAC9C,OAAO,EAAE,IAAI,EAAE,MAAM,wBAAwB,CAAC;AAC9C,YAAY,EAAE,IAAI,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AACrD,OAAO,EAAE,YAAY,EAAE,MAAM,gCAAgC,CAAC;AAC9D,YAAY,EACX,iBAAiB,EACjB,mBAAmB,EACnB,YAAY,GACZ,MAAM,wBAAwB,CAAC;AAChC,OAAO,EAAE,IAAI,EAAE,MAAM,wBAAwB,CAAC;AAC9C,OAAO,EAAE,IAAI,EAAE,MAAM,wBAAwB,CAAC;AAC9C,YAAY,EAAE,IAAI,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@project-ajax/sdk",
-	"version": "0.0.56",
+	"version": "0.0.59",
 	"description": "An SDK for building workers for the Project Ajax platform",
 	"license": "UNLICENSED",
 	"type": "module",

package/src/capabilities/sync.ts

@@ -20,11 +20,40 @@ export type SyncedObject<PK extends string, S extends PropertySchema<PK>> = {
 	pageContentMarkdown?: string;
 };
 
+/**
+ * Result returned from the sync execute function.
+ */
+export type SyncExecutionResult<PK extends string, Context = unknown> = {
+	/**
+	 * The batch of objects fetched in this execution.
+	 */
+	objects: SyncedObject<PK, PropertySchema<PK>>[];
+
+	/**
+	 * Indicates whether the sync is complete.
+	 * - `true`: No more data to fetch, sync is complete
+	 * - `false`: More data available, will trigger another execution with nextContext
+	 */
+	done: boolean;
+
+	/**
+	 * Optional context data to pass to the next execution.
+	 * Required if `done` is `false`, ignored if `done` is `true`.
+	 * This can be any type of data (cursor, page number, timestamp, etc.).
+	 * The same data will be provided in the context parameter of the next execution.
+	 */
+	nextContext?: Context;
+};
+
 /**
  * A configuration object that enables synchronization between a data
  * source and a third-party source.
  */
-export type SyncConfiguration<PK extends string, S extends Schema<PK>> = {
+export type SyncConfiguration<
+	PK extends string,
+	S extends Schema<PK>,
+	Context = unknown,
+> = {
 	/**
 	 * The property of the data source that maps to a "primary key" in the
 	 * third-party data. This is used to match existing pages to
@@ -37,19 +66,40 @@ export type SyncConfiguration<PK extends string, S extends Schema<PK>> = {
 	 */
 	schema: S;
 
+	/**
+	 * Whether to delete pages from the collection that are not returned
+	 * from any of the sync executions (mark-and-sweep deletion).
+	 *
+	 * - `true`: Pages not seen in any execution batch will be deleted
+	 * - `false` (default): Only upsert pages, never delete
+	 *
+	 * @default false
+	 */
+	deleteUnreturnedPages?: boolean;
+
 	/**
 	 * A function that fetches the data to sync from the third-party service.
 	 *
-	 * For now, this function must return all of the data to be synced each time
-	 * it's called. The runtime will handle diffing this against the data source
-	 * and creating, updating, and deleting pages as necessary.
+	 * This function can return all data at once, or implement pagination by:
+	 * 1. Returning a batch of objects with `done: false` and a `nextContext`
+	 * 2. The runtime will call execute again with that context data
+	 * 3. Continue until `done: true` is returned
+	 *
+	 * The runtime will handle diffing against the data source and creating,
+	 * updating, and deleting pages as necessary.
+	 *
+	 * @param context - Optional context data from previous execution (undefined on first call)
+	 * @returns A result containing objects, done status, and optional nextContext
 	 */
-	execute: () => Promise<SyncedObject<PK, PropertySchema<PK>>[]>;
+	execute: (context?: Context) => Promise<SyncExecutionResult<PK, Context>>;
 };
 
-export type SyncHandlerResult<PK extends string> = {
+export type SyncHandlerResult<PK extends string, Context = unknown> = {
 	primaryKeyProperty: PK;
 	objects: SyncedObject<PK, PropertySchema<PK>>[];
+	done: boolean;
+	nextContext?: Context;
+	deleteUnreturnedPages?: boolean;
 };
 
 /**
@@ -59,23 +109,29 @@ export type SyncHandlerResult<PK extends string> = {
  * @returns A handler function that executes the sync function, and passes data
  * needed to complete the sync back to the platform.
  */
-export function sync<PK extends string, S extends Schema<PK>>(
-	syncConfiguration: SyncConfiguration<PK, S>,
-) {
+export function sync<
+	PK extends string,
+	S extends Schema<PK>,
+	Context = unknown,
+>(syncConfiguration: SyncConfiguration<PK, S, Context>) {
 	return {
 		_tag: "sync",
 		config: {
+			primaryKeyProperty: syncConfiguration.primaryKeyProperty,
 			schema: syncConfiguration.schema,
+			deleteUnreturnedPages: syncConfiguration.deleteUnreturnedPages,
 		},
-		async handler() {
-			const objects = await syncConfiguration.execute().catch((err) => {
-				throw new ExecutionError(err);
-			});
+		async handler(context?: Context) {
+			const executionResult = await syncConfiguration
+				.execute(context)
+				.catch((err) => {
+					throw new ExecutionError(err);
+				});
 
 			const result = {
-				schema: syncConfiguration.schema,
-				primaryKeyProperty: syncConfiguration.primaryKeyProperty,
-				objects,
+				objects: executionResult.objects,
+				done: executionResult.done,
+				nextContext: executionResult.nextContext,
 			};
 
 			process.stdout.write(`\n<output>${JSON.stringify(result)}</output>\n`);

package/src/cli/commands/auth.impl.test.ts

@@ -125,9 +125,11 @@ describe("login", () => {
 			configFile: {
 				token: null,
 				workerId: null,
-				environment: "prod",
 				baseUrl: "https://www.notion.so",
 			},
+			flags: {
+				env: "local",
+			},
 		});
 
 		vi.spyOn(Config, "load").mockResolvedValue(mockConfig);
@@ -142,9 +144,9 @@ describe("login", () => {
 		await login.call(context, baseFlags, testToken);
 
 		expect(updateSpy).toHaveBeenCalledWith({
-			environment: "prod",
 			token: testToken,
 			workerId: null,
+			environment: "local",
 		});
 	});
 });

package/src/cli/commands/auth.impl.ts

@@ -1,3 +1,4 @@
+import type { ConfigMap } from "../config.js";
 import type { GlobalFlags } from "../flags.js";
 import { buildHandler, type HandlerContext } from "../handler.js";
 import { openNotionUrl } from "../utils/openUrl.js";
@@ -28,11 +29,18 @@ export const login = buildHandler(async function (
 		return;
 	}
 
-	await this.config.update({
-		environment,
+	const update: Partial<ConfigMap> = {
 		token,
 		workerId: null,
-	});
+	};
+
+	console.log("environment", environment);
+
+	if (environment !== "prod") {
+		update.environment = environment;
+	}
+
+	await this.config.update(update);
 
 	this.io.writeErr("Successfully logged in!");
 });

package/src/cli/config.ts

@@ -141,9 +141,8 @@ export class Config {
 			partialConfig.workerId = opts.flags["worker-id"];
 		}
 
-		partialConfig.baseUrl ??= baseUrlForEnvironment(
-			partialConfig.environment ?? "prod",
-		);
+		partialConfig.environment ??= "prod";
+		partialConfig.baseUrl ??= baseUrlForEnvironment(partialConfig.environment);
 
 		const environment = partialConfig.environment;
 		if (!environment) {
@@ -194,8 +193,6 @@ export class Config {
 				configFile = {
 					token: null,
 					workerId: null,
-					environment: "prod",
-					baseUrl: baseUrlForEnvironment("prod"),
 				};
 			} else {
 				throw error;

package/src/cli/io.ts

@@ -1,4 +1,6 @@
+// biome-ignore lint/style/noRestrictedImports: This file contains safe wrappers for prompts.
 import * as prompts from "@inquirer/prompts";
+// biome-ignore lint/style/noRestrictedImports: This is the one allowed location for this import.
 import { createTable, type TableCell, type TableItem } from "@visulima/tabular";
 
 export interface WriterOptions {
@@ -80,21 +82,22 @@ export class IO {
 	}
 
 	confirm(config: WithSafety<Parameters<typeof prompts.confirm>[0]>) {
-		if (!process.stdin.isTTY) {
-			this.writeErr(config.noTTY);
-			process.exit(1);
-		}
-
+		this.#ensureTTY(config.noTTY);
 		return prompts.confirm(config).catch(this.#handlePromptExit.bind(this));
 	}
 
 	input(config: WithSafety<Parameters<typeof prompts.input>[0]>) {
+		this.#ensureTTY(config.noTTY);
+		return prompts.input(config).catch(this.#handlePromptExit.bind(this));
+	}
+
+	#ensureTTY(noTTY: string): true | never {
 		if (!process.stdin.isTTY) {
-			this.writeErr(config.noTTY);
+			this.writeErr(noTTY);
 			process.exit(1);
 		}
 
-		return prompts.input(config).catch(this.#handlePromptExit.bind(this));
+		return true;
 	}
 
 	#handlePromptExit(err: unknown) {

package/src/index.ts

@@ -1,5 +1,10 @@
 export { emojiIcon, notionIcon } from "./builder.js";
 export { slashCommand } from "./capabilities/slashCommand.js";
+export type {
+	SyncConfiguration,
+	SyncExecutionResult,
+	SyncedObject,
+} from "./capabilities/sync.js";
 export { sync } from "./capabilities/sync.js";
 export { tool } from "./capabilities/tool.js";
 export type { Icon, NoticonColor, NoticonName } from "./types.js";