@filoz/repair-cli 0.3.0 → 1.0.0

package/dist/src/db/second-copy-create.js

@@ -0,0 +1,104 @@
+import { readFile } from 'node:fs/promises';
+import { taskLog } from '@clack/prompts';
+import { getBlockNumber } from 'viem/actions';
+import { NoAlternateProviderError, RepairCreationError } from "../error.js";
+import { getRepairProvider } from "./get-repair-provider.js";
+const OPERATIONS_INSERT_BATCH_SIZE = 500;
+export function normalizeProviderUrl(url) {
+    let parsed;
+    try {
+        parsed = new URL(url);
+    }
+    catch {
+        throw new RepairCreationError(`Invalid source provider URL: ${url}`);
+    }
+    if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') {
+        throw new RepairCreationError(`Source provider URL must be http(s): ${url}`);
+    }
+    return parsed.toString().replace(/\/$/, '');
+}
+export async function parsePieceCidsFromFile(piecesFile) {
+    let raw;
+    try {
+        raw = JSON.parse(await readFile(piecesFile, 'utf8'));
+    }
+    catch (error) {
+        throw new RepairCreationError(error instanceof Error ? `Failed to read pieces file: ${error.message}` : 'Failed to read pieces file');
+    }
+    if (raw == null || typeof raw !== 'object' || !Array.isArray(raw.aggregates)) {
+        throw new RepairCreationError('Pieces file must contain an "aggregates" array');
+    }
+    const cids = new Set();
+    for (const aggregate of raw.aggregates) {
+        if (typeof aggregate?.piece_cid !== 'string' || aggregate.piece_cid.length === 0) {
+            throw new RepairCreationError('Each aggregate must include a non-empty "piece_cid" string');
+        }
+        cids.add(aggregate.piece_cid);
+    }
+    if (cids.size === 0) {
+        throw new RepairCreationError('Pieces file contains no piece CIDs');
+    }
+    return [...cids];
+}
+export async function secondCopyCreate(options) {
+    const { api, localDb, piecesFile, sourceProviderUrl, targetProviderId, client } = options;
+    const localSchema = localDb._.fullSchema;
+    const now = Date.now();
+    const blockNumber = await getBlockNumber(client);
+    const normalizedSourceUrl = normalizeProviderUrl(sourceProviderUrl);
+    const log = taskLog({
+        title: 'Creating second-copy',
+        limit: 10,
+        retainLog: true,
+    });
+    const targetProvider = await getRepairProvider({
+        api,
+        providerId: targetProviderId,
+    });
+    if (!targetProvider) {
+        throw new NoAlternateProviderError(targetProviderId);
+    }
+    const normalizedTargetUrl = normalizeProviderUrl(targetProvider.serviceUrl);
+    if (normalizedSourceUrl === normalizedTargetUrl) {
+        throw new RepairCreationError('Source provider URL must differ from the target provider URL');
+    }
+    const pieceCids = await parsePieceCidsFromFile(piecesFile);
+    log.message(`Loaded ${pieceCids.length} unique piece CIDs from ${piecesFile}`);
+    const [repair] = await localDb
+        .insert(localSchema.repairs)
+        .values({
+        kind: 'second_copy',
+        repairProviderId: 0n,
+        repairDataSetId: null,
+        sourceProviderUrl: normalizedSourceUrl,
+        targetProviderId: targetProvider.providerId,
+        targetProviderUrl: targetProvider.serviceUrl,
+        targetDataSetId: null,
+        blockNumber,
+        createdAt: now,
+        updatedAt: now,
+    })
+        .returning({ id: localSchema.repairs.id });
+    if (!repair)
+        throw new RepairCreationError('Failed to create second-copy row');
+    let totalOperations = 0;
+    for (let offset = 0; offset < pieceCids.length; offset += OPERATIONS_INSERT_BATCH_SIZE) {
+        const batchCids = pieceCids.slice(offset, offset + OPERATIONS_INSERT_BATCH_SIZE);
+        const operations = batchCids.map((cid) => ({
+            repairId: repair.id,
+            type: 'add_piece',
+            status: 'pending',
+            cid,
+            metadata: {},
+            alternateProvider: normalizedSourceUrl,
+            createdAt: now,
+            updatedAt: now,
+        }));
+        await localDb.insert(localSchema.operations).values(operations);
+        totalOperations += operations.length;
+        log.message(`Inserted ${operations.length} operations (${totalOperations}/${pieceCids.length})`);
+    }
+    log.success(`Created second-copy ${repair.id} with ${totalOperations} operations`, { showLog: true });
+    return repair.id;
+}
+//# sourceMappingURL=second-copy-create.js.map

package/dist/src/db/second-copy-create.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"second-copy-create.js","sourceRoot":"","sources":["../../../src/db/second-copy-create.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAA;AAC3C,OAAO,EAAE,OAAO,EAAE,MAAM,gBAAgB,CAAA;AACxC,OAAO,EAAE,cAAc,EAAE,MAAM,cAAc,CAAA;AAC7C,OAAO,EAAE,wBAAwB,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAA;AAG3E,OAAO,EAAE,iBAAiB,EAAE,MAAM,0BAA0B,CAAA;AAE5D,MAAM,4BAA4B,GAAG,GAAG,CAAA;AAexC,MAAM,UAAU,oBAAoB,CAAC,GAAW;IAC9C,IAAI,MAAW,CAAA;IACf,IAAI,CAAC;QACH,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAA;IACvB,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,IAAI,mBAAmB,CAAC,gCAAgC,GAAG,EAAE,CAAC,CAAA;IACtE,CAAC;IAED,IAAI,MAAM,CAAC,QAAQ,KAAK,OAAO,IAAI,MAAM,CAAC,QAAQ,KAAK,QAAQ,EAAE,CAAC;QAChE,MAAM,IAAI,mBAAmB,CAAC,wCAAwC,GAAG,EAAE,CAAC,CAAA;IAC9E,CAAC;IAGD,OAAO,MAAM,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAA;AAC7C,CAAC;AAKD,MAAM,CAAC,KAAK,UAAU,sBAAsB,CAAC,UAAkB;IAC7D,IAAI,GAAY,CAAA;IAChB,IAAI,CAAC;QACH,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC,CAAA;IACtD,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,IAAI,mBAAmB,CAC3B,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,+BAA+B,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,4BAA4B,CACvG,CAAA;IACH,CAAC;IAED,IAAI,GAAG,IAAI,IAAI,IAAI,OAAO,GAAG,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAE,GAAkB,CAAC,UAAU,CAAC,EAAE,CAAC;QAC7F,MAAM,IAAI,mBAAmB,CAAC,gDAAgD,CAAC,CAAA;IACjF,CAAC;IAED,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAA;IAC9B,KAAK,MAAM,SAAS,IAAK,GAAkB,CAAC,UAAU,EAAE,CAAC;QACvD,IAAI,OAAO,SAAS,EAAE,SAAS,KAAK,QAAQ,IAAI,SAAS,CAAC,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACjF,MAAM,IAAI,mBAAmB,CAAC,4DAA4D,CAAC,CAAA;QAC7F,CAAC;QACD,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,SAAS,CAAC,CAAA;IAC/B,CAAC;IAED,IAAI,IAAI,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;QACpB,MAAM,IAAI,mBAAmB,CAAC,oCAAoC,CAAC,CAAA;IACrE,CAAC;IAED,OAAO,CAAC,GAAG,IAAI,CAAC,CAAA;AAClB,CAAC;AAQD,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,OAAgC;IACrE,MAAM,EAAE,GAAG,EAAE,OAAO,EAAE,UAAU,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,MAAM,EAAE,GAAG,OAAO,CAAA;IACzF,MAAM,WAAW,GAAG,OAAO,CAAC,CAAC,CAAC,UAAU,CAAA;IACxC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;IACtB,MAAM,WAAW,GAAG,MAAM,cAAc,CAAC,MAAM,CAAC,CAAA;IAChD,MAAM,mBAAmB,GAAG,oBAAoB,CAAC,iBAAiB,CAAC,CAAA;IAEnE,MAAM,GAAG,GAAG,OAAO,CAAC;QAClB,KAAK,EAAE,sBAAsB;QAC7B,KAAK,EAAE,EAAE;QACT,SAAS,EAAE,IAAI;KAChB,CAAC,CAAA;IAEF,MAAM,cAAc,GAAG,MAAM,iBAAiB,CAAC;QAC7C,GAAG;QACH,UAAU,EAAE,gBAAgB;KAC7B,CAAC,CAAA;IAEF,IAAI,CAAC,cAAc,EAAE,CAAC;QACpB,MAAM,IAAI,wBAAwB,CAAC,gBAAgB,CAAC,CAAA;IACtD,CAAC;IAED,MAAM,mBAAmB,GAAG,oBAAoB,CAAC,cAAc,CAAC,UAAU,CAAC,CAAA;IAC3E,IAAI,mBAAmB,KAAK,mBAAmB,EAAE,CAAC;QAChD,MAAM,IAAI,mBAAmB,CAAC,8DAA8D,CAAC,CAAA;IAC/F,CAAC;IAED,MAAM,SAAS,GAAG,MAAM,sBAAsB,CAAC,UAAU,CAAC,CAAA;IAC1D,GAAG,CAAC,OAAO,CAAC,UAAU,SAAS,CAAC,MAAM,2BAA2B,UAAU,EAAE,CAAC,CAAA;IAE9E,MAAM,CAAC,MAAM,CAAC,GAAG,MAAM,OAAO;SAC3B,MAAM,CAAC,WAAW,CAAC,OAAO,CAAC;SAC3B,MAAM,CAAC;QACN,IAAI,EAAE,aAAa;QACnB,gBAAgB,EAAE,EAAE;QACpB,eAAe,EAAE,IAAI;QACrB,iBAAiB,EAAE,mBAAmB;QACtC,gBAAgB,EAAE,cAAc,CAAC,UAAU;QAC3C,iBAAiB,EAAE,cAAc,CAAC,UAAU;QAC5C,eAAe,EAAE,IAAI;QACrB,WAAW;QACX,SAAS,EAAE,GAAG;QACd,SAAS,EAAE,GAAG;KACf,CAAC;SACD,SAAS,CAAC,EAAE,EAAE,EAAE,WAAW,CAAC,OAAO,CAAC,EAAE,EAAE,CAAC,CAAA;IAE5C,IAAI,CAAC,MAAM;QAAE,MAAM,IAAI,mBAAmB,CAAC,kCAAkC,CAAC,CAAA;IAE9E,IAAI,eAAe,GAAG,CAAC,CAAA;IACvB,KAAK,IAAI,MAAM,GAAG,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC,MAAM,EAAE,MAAM,IAAI,4BAA4B,EAAE,CAAC;QACvF,MAAM,SAAS,GAAG,SAAS,CAAC,KAAK,CAAC,MAAM,EAAE,MAAM,GAAG,4BAA4B,CAAC,CAAA;QAChF,MAAM,UAAU,GAAsB,SAAS,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;YAC5D,QAAQ,EAAE,MAAM,CAAC,EAAE;YACnB,IAAI,EAAE,WAAW;YACjB,MAAM,EAAE,SAAS;YACjB,GAAG;YACH,QAAQ,EAAE,EAAE;YACZ,iBAAiB,EAAE,mBAAmB;YACtC,SAAS,EAAE,GAAG;YACd,SAAS,EAAE,GAAG;SACf,CAAC,CAAC,CAAA;QAEH,MAAM,OAAO,CAAC,MAAM,CAAC,WAAW,CAAC,UAAU,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC,CAAA;QAC/D,eAAe,IAAI,UAAU,CAAC,MAAM,CAAA;QACpC,GAAG,CAAC,OAAO,CAAC,YAAY,UAAU,CAAC,MAAM,gBAAgB,eAAe,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,CAAA;IAClG,CAAC;IAED,GAAG,CAAC,OAAO,CAAC,uBAAuB,MAAM,CAAC,EAAE,SAAS,eAAe,aAAa,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAA;IACrG,OAAO,MAAM,CAAC,EAAE,CAAA;AAClB,CAAC"}

package/dist/src/local-schema.d.ts

@@ -1,6 +1,7 @@
 import type { MetadataObject } from '@filoz/synapse-core';
 import * as t from 'drizzle-orm/sqlite-core';
 export type RepairStatus = 'pending' | 'completed' | 'failed';
+export type RepairKind = 'repair' | 'replicate' | 'second_copy';
 export type OperationStatus = 'pending' | 'completed' | 'failed' | 'skipped';
 export type OperationType = 'create_dataset' | 'add_piece';
 export declare const jsonType: {
@@ -99,6 +100,26 @@ export declare const repairs: t.SQLiteTableWithColumns<{
             length: undefined;
             $type: RepairStatus;
         }>;
+        kind: t.SQLiteColumn<{
+            name: "kind";
+            tableName: "repairs";
+            dataType: "string";
+            columnType: "SQLiteText";
+            data: RepairKind;
+            driverParam: string;
+            notNull: true;
+            hasDefault: true;
+            isPrimaryKey: false;
+            isAutoincrement: false;
+            hasRuntimeDefault: false;
+            enumValues: [string, ...string[]];
+            baseColumn: never;
+            identity: undefined;
+            generated: undefined;
+        }, {}, {
+            length: undefined;
+            $type: RepairKind;
+        }>;
         repairProviderId: t.SQLiteColumn<{
             name: "repair_provider_id";
             tableName: "repairs";
@@ -137,6 +158,25 @@ export declare const repairs: t.SQLiteTableWithColumns<{
         }, {}, {
             sqliteColumnBuilderBrand: "SQLiteCustomColumnBuilderBrand";
         }>;
+        sourceProviderUrl: t.SQLiteColumn<{
+            name: "source_provider_url";
+            tableName: "repairs";
+            dataType: "string";
+            columnType: "SQLiteText";
+            data: string;
+            driverParam: string;
+            notNull: false;
+            hasDefault: false;
+            isPrimaryKey: false;
+            isAutoincrement: false;
+            hasRuntimeDefault: false;
+            enumValues: [string, ...string[]];
+            baseColumn: never;
+            identity: undefined;
+            generated: undefined;
+        }, {}, {
+            length: number | undefined;
+        }>;
         targetProviderId: t.SQLiteColumn<{
             name: "target_provider_id";
             tableName: "repairs";
@@ -366,8 +406,8 @@ export declare const operations: t.SQLiteTableWithColumns<{
             identity: undefined;
             generated: undefined;
         }, {}, {
-            $type: MetadataObject;
             sqliteColumnBuilderBrand: "SQLiteCustomColumnBuilderBrand";
+            $type: MetadataObject;
         }>;
         alternateProvider: t.SQLiteColumn<{
             name: "alternate_provider";

package/dist/src/local-schema.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"local-schema.d.ts","sourceRoot":"","sources":["../../src/local-schema.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAA;AAGzD,OAAO,KAAK,CAAC,MAAM,yBAAyB,CAAA;AAI5C,MAAM,MAAM,YAAY,GAAG,SAAS,GAAG,WAAW,GAAG,QAAQ,CAAA;AAC7D,MAAM,MAAM,eAAe,GAAG,SAAS,GAAG,WAAW,GAAG,QAAQ,GAAG,SAAS,CAAA;AAC5E,MAAM,MAAM,aAAa,GAAG,gBAAgB,GAAG,WAAW,CAAA;AAM1D,eAAO,MAAM,QAAQ;;;;;;;;;;;;;;;;;;;;;;;;;CAUnB,CAAA;AAEF,eAAO,MAAM,UAAU;;;;;;;;;;;;;;;;;;;;;;;;;CAUrB,CAAA;AAEF,MAAM,MAAM,YAAY,GAAG,OAAO,OAAO,CAAC,YAAY,CAAA;AACtD,MAAM,MAAM,YAAY,GAAG,OAAO,OAAO,CAAC,YAAY,CAAA;AACtD,MAAM,MAAM,YAAY,GAAG,OAAO,CAAC,YAAY,CAAC,CAAA;AAEhD,eAAO,MAAM,OAAO;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAWlB,CAAA;AAEF,MAAM,MAAM,eAAe,GAAG,OAAO,UAAU,CAAC,YAAY,CAAA;AAC5D,MAAM,MAAM,eAAe,GAAG,OAAO,UAAU,CAAC,YAAY,CAAA;AAE5D,eAAO,MAAM,UAAU;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAerB,CAAA;AAEF,eAAO,MAAM,eAAe;;EAEzB,CAAA;AAEH,eAAO,MAAM,kBAAkB;;EAK5B,CAAA"}
+{"version":3,"file":"local-schema.d.ts","sourceRoot":"","sources":["../../src/local-schema.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAA;AAGzD,OAAO,KAAK,CAAC,MAAM,yBAAyB,CAAA;AAI5C,MAAM,MAAM,YAAY,GAAG,SAAS,GAAG,WAAW,GAAG,QAAQ,CAAA;AAC7D,MAAM,MAAM,UAAU,GAAG,QAAQ,GAAG,WAAW,GAAG,aAAa,CAAA;AAC/D,MAAM,MAAM,eAAe,GAAG,SAAS,GAAG,WAAW,GAAG,QAAQ,GAAG,SAAS,CAAA;AAC5E,MAAM,MAAM,aAAa,GAAG,gBAAgB,GAAG,WAAW,CAAA;AAM1D,eAAO,MAAM,QAAQ;;;;;;;;;;;;;;;;;;;;;;;;;CAUnB,CAAA;AAEF,eAAO,MAAM,UAAU;;;;;;;;;;;;;;;;;;;;;;;;;CAUrB,CAAA;AAEF,MAAM,MAAM,YAAY,GAAG,OAAO,OAAO,CAAC,YAAY,CAAA;AACtD,MAAM,MAAM,YAAY,GAAG,OAAO,OAAO,CAAC,YAAY,CAAA;AACtD,MAAM,MAAM,YAAY,GAAG,OAAO,CAAC,YAAY,CAAC,CAAA;AAEhD,eAAO,MAAM,OAAO;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAalB,CAAA;AAEF,MAAM,MAAM,eAAe,GAAG,OAAO,UAAU,CAAC,YAAY,CAAA;AAC5D,MAAM,MAAM,eAAe,GAAG,OAAO,UAAU,CAAC,YAAY,CAAA;AAE5D,eAAO,MAAM,UAAU;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAerB,CAAA;AAEF,eAAO,MAAM,eAAe;;EAEzB,CAAA;AAEH,eAAO,MAAM,kBAAkB;;EAK5B,CAAA"}

package/dist/src/local-schema.js

@@ -27,8 +27,10 @@ export const bigintType = customType({
 export const repairs = table('repairs', {
     id: t.int().primaryKey({ autoIncrement: true }),
     status: t.text().$type().notNull().default('pending'),
+    kind: t.text().$type().notNull().default('repair'),
     repairProviderId: bigintType('repair_provider_id').notNull(),
     repairDataSetId: bigintType('repair_data_set_id'),
+    sourceProviderUrl: t.text('source_provider_url'),
     targetProviderId: bigintType('target_provider_id').notNull(),
     targetProviderUrl: t.text('target_provider_url').notNull(),
     targetDataSetId: bigintType('target_data_set_id'),

package/dist/src/local-schema.js.map

@@ -1 +1 @@
-{"version":3,"file":"local-schema.js","sourceRoot":"","sources":["../../src/local-schema.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAA;AAEvC,OAAO,KAAK,CAAC,MAAM,yBAAyB,CAAA;AAC5C,OAAO,EAAE,UAAU,EAAE,WAAW,IAAI,KAAK,EAAE,MAAM,yBAAyB,CAAA;AAC1E,OAAO,KAAK,IAAI,MAAM,eAAe,CAAA;AAUrC,MAAM,CAAC,MAAM,QAAQ,GAAG,UAAU,CAAoB;IACpD,QAAQ;QACN,OAAO,MAAM,CAAA;IACf,CAAC;IACD,QAAQ,CAAC,KAAK;QACZ,OAAO,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAA;IAC9B,CAAC;IACD,UAAU,CAAC,KAAK;QACd,OAAO,IAAI,CAAC,KAAK,CAAC,KAAe,CAAC,CAAA;IACpC,CAAC;CACF,CAAC,CAAA;AAEF,MAAM,CAAC,MAAM,UAAU,GAAG,UAAU,CAAmB;IACrD,QAAQ;QACN,OAAO,MAAM,CAAA;IACf,CAAC;IACD,QAAQ,CAAC,KAAK;QACZ,OAAO,KAAK,CAAC,QAAQ,EAAE,CAAA;IACzB,CAAC;IACD,UAAU,CAAC,KAAK;QACd,OAAO,MAAM,CAAC,KAAe,CAAC,CAAA;IAChC,CAAC;CACF,CAAC,CAAA;AAMF,MAAM,CAAC,MAAM,OAAO,GAAG,KAAK,CAAC,SAAS,EAAE;IACtC,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC;IAC/C,MAAM,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,KAAK,EAAgB,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,SAAS,CAAC;IACnE,gBAAgB,EAAE,UAAU,CAAC,oBAAoB,CAAC,CAAC,OAAO,EAAE;IAC5D,eAAe,EAAE,UAAU,CAAC,oBAAoB,CAAC;IACjD,gBAAgB,EAAE,UAAU,CAAC,oBAAoB,CAAC,CAAC,OAAO,EAAE;IAC5D,iBAAiB,EAAE,CAAC,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC,OAAO,EAAE;IAC1D,eAAe,EAAE,UAAU,CAAC,oBAAoB,CAAC;IACjD,WAAW,EAAE,UAAU,CAAC,cAAc,CAAC,CAAC,OAAO,EAAE;IACjD,SAAS,EAAE,CAAC,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC,OAAO,EAAE;IAC5C,SAAS,EAAE,CAAC,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC,OAAO,EAAE;CAC7C,CAAC,CAAA;AAKF,MAAM,CAAC,MAAM,UAAU,GAAG,KAAK,CAAC,YAAY,EAAE;IAC5C,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC;IAC/C,QAAQ,EAAE,CAAC;SACR,GAAG,CAAC,WAAW,CAAC;SAChB,UAAU,CAAC,GAAoB,EAAE,CAAC,OAAO,CAAC,EAAE,CAAC;SAC7C,OAAO,EAAE;IACZ,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,KAAK,EAAiB,CAAC,OAAO,EAAE;IAC/C,MAAM,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,KAAK,EAAmB,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,SAAS,CAAC;IACtE,GAAG,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,OAAO,EAAE;IACvB,QAAQ,EAAE,QAAQ,EAAE,CAAC,KAAK,EAAkB,CAAC,OAAO,EAAE;IACtD,iBAAiB,EAAE,CAAC,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC,OAAO,EAAE;IACzD,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC;IACzB,KAAK,EAAE,CAAC,CAAC,IAAI,EAAE;IACf,SAAS,EAAE,CAAC,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC,OAAO,EAAE;IAC5C,SAAS,EAAE,CAAC,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC,OAAO,EAAE;CAC7C,CAAC,CAAA;AAEF,MAAM,CAAC,MAAM,eAAe,GAAG,SAAS,CAAC,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC;IAC/D,UAAU,EAAE,IAAI,CAAC,UAAU,CAAC;CAC7B,CAAC,CAAC,CAAA;AAEH,MAAM,CAAC,MAAM,kBAAkB,GAAG,SAAS,CAAC,UAAU,EAAE,CAAC,EAAE,GAAG,EAAE,EAAE,EAAE,CAAC,CAAC;IACpE,MAAM,EAAE,GAAG,CAAC,OAAO,EAAE;QACnB,MAAM,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC;QAC7B,UAAU,EAAE,CAAC,OAAO,CAAC,EAAE,CAAC;KACzB,CAAC;CACH,CAAC,CAAC,CAAA"}
+{"version":3,"file":"local-schema.js","sourceRoot":"","sources":["../../src/local-schema.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAA;AAEvC,OAAO,KAAK,CAAC,MAAM,yBAAyB,CAAA;AAC5C,OAAO,EAAE,UAAU,EAAE,WAAW,IAAI,KAAK,EAAE,MAAM,yBAAyB,CAAA;AAC1E,OAAO,KAAK,IAAI,MAAM,eAAe,CAAA;AAWrC,MAAM,CAAC,MAAM,QAAQ,GAAG,UAAU,CAAoB;IACpD,QAAQ;QACN,OAAO,MAAM,CAAA;IACf,CAAC;IACD,QAAQ,CAAC,KAAK;QACZ,OAAO,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAA;IAC9B,CAAC;IACD,UAAU,CAAC,KAAK;QACd,OAAO,IAAI,CAAC,KAAK,CAAC,KAAe,CAAC,CAAA;IACpC,CAAC;CACF,CAAC,CAAA;AAEF,MAAM,CAAC,MAAM,UAAU,GAAG,UAAU,CAAmB;IACrD,QAAQ;QACN,OAAO,MAAM,CAAA;IACf,CAAC;IACD,QAAQ,CAAC,KAAK;QACZ,OAAO,KAAK,CAAC,QAAQ,EAAE,CAAA;IACzB,CAAC;IACD,UAAU,CAAC,KAAK;QACd,OAAO,MAAM,CAAC,KAAe,CAAC,CAAA;IAChC,CAAC;CACF,CAAC,CAAA;AAMF,MAAM,CAAC,MAAM,OAAO,GAAG,KAAK,CAAC,SAAS,EAAE;IACtC,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC;IAC/C,MAAM,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,KAAK,EAAgB,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,SAAS,CAAC;IACnE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,KAAK,EAAc,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,QAAQ,CAAC;IAC9D,gBAAgB,EAAE,UAAU,CAAC,oBAAoB,CAAC,CAAC,OAAO,EAAE;IAC5D,eAAe,EAAE,UAAU,CAAC,oBAAoB,CAAC;IACjD,iBAAiB,EAAE,CAAC,CAAC,IAAI,CAAC,qBAAqB,CAAC;IAChD,gBAAgB,EAAE,UAAU,CAAC,oBAAoB,CAAC,CAAC,OAAO,EAAE;IAC5D,iBAAiB,EAAE,CAAC,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC,OAAO,EAAE;IAC1D,eAAe,EAAE,UAAU,CAAC,oBAAoB,CAAC;IACjD,WAAW,EAAE,UAAU,CAAC,cAAc,CAAC,CAAC,OAAO,EAAE;IACjD,SAAS,EAAE,CAAC,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC,OAAO,EAAE;IAC5C,SAAS,EAAE,CAAC,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC,OAAO,EAAE;CAC7C,CAAC,CAAA;AAKF,MAAM,CAAC,MAAM,UAAU,GAAG,KAAK,CAAC,YAAY,EAAE;IAC5C,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC;IAC/C,QAAQ,EAAE,CAAC;SACR,GAAG,CAAC,WAAW,CAAC;SAChB,UAAU,CAAC,GAAoB,EAAE,CAAC,OAAO,CAAC,EAAE,CAAC;SAC7C,OAAO,EAAE;IACZ,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,KAAK,EAAiB,CAAC,OAAO,EAAE;IAC/C,MAAM,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,KAAK,EAAmB,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,SAAS,CAAC;IACtE,GAAG,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,OAAO,EAAE;IACvB,QAAQ,EAAE,QAAQ,EAAE,CAAC,KAAK,EAAkB,CAAC,OAAO,EAAE;IACtD,iBAAiB,EAAE,CAAC,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC,OAAO,EAAE;IACzD,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC;IACzB,KAAK,EAAE,CAAC,CAAC,IAAI,EAAE;IACf,SAAS,EAAE,CAAC,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC,OAAO,EAAE;IAC5C,SAAS,EAAE,CAAC,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC,OAAO,EAAE;CAC7C,CAAC,CAAA;AAEF,MAAM,CAAC,MAAM,eAAe,GAAG,SAAS,CAAC,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC;IAC/D,UAAU,EAAE,IAAI,CAAC,UAAU,CAAC;CAC7B,CAAC,CAAC,CAAA;AAEH,MAAM,CAAC,MAAM,kBAAkB,GAAG,SAAS,CAAC,UAAU,EAAE,CAAC,EAAE,GAAG,EAAE,EAAE,EAAE,CAAC,CAAC;IACpE,MAAM,EAAE,GAAG,CAAC,OAAO,EAAE;QACnB,MAAM,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC;QAC7B,UAAU,EAAE,CAAC,OAAO,CAAC,EAAE,CAAC;KACzB,CAAC;CACH,CAAC,CAAC,CAAA"}

package/dist/src/middleware.d.ts

@@ -1,21 +1,19 @@
 import type { Chain } from '@filoz/synapse-core/chains';
 import { z } from 'incur';
 import type { Account, Client, Transport } from 'viem';
-import type { IndexerDatabase, LocalDatabase } from './types.ts';
+import type { IndexerApi, LocalDatabase } from './types.ts';
 export declare const contextSchema: z.ZodObject<{
-    indexerDb: z.ZodCustom<IndexerDatabase, IndexerDatabase>;
+    api: z.ZodCustom<IndexerApi, IndexerApi>;
     localDb: z.ZodCustom<LocalDatabase, LocalDatabase>;
     config: z.ZodCustom<import("iso-conf").Conf<z.ZodObject<{
         privateKey: z.ZodOptional<z.ZodString>;
-        indexerMainnetUrl: z.ZodOptional<z.ZodURL>;
-        indexerCalibrationUrl: z.ZodOptional<z.ZodURL>;
+        apiUrl: z.ZodOptional<z.ZodURL>;
         chainId: z.ZodOptional<z.ZodNumber>;
         dbPath: z.ZodOptional<z.ZodString>;
         source: z.ZodOptional<z.ZodString>;
     }, z.core.$strip>>, import("iso-conf").Conf<z.ZodObject<{
         privateKey: z.ZodOptional<z.ZodString>;
-        indexerMainnetUrl: z.ZodOptional<z.ZodURL>;
-        indexerCalibrationUrl: z.ZodOptional<z.ZodURL>;
+        apiUrl: z.ZodOptional<z.ZodURL>;
         chainId: z.ZodOptional<z.ZodNumber>;
         dbPath: z.ZodOptional<z.ZodString>;
         source: z.ZodOptional<z.ZodString>;
@@ -26,19 +24,17 @@ export declare const contextSchema: z.ZodObject<{
     isInteractive: z.ZodBoolean;
 }, z.core.$strip>;
 export declare const contextMiddleware: import("incur").MiddlewareHandler<z.ZodObject<{
-    indexerDb: z.ZodCustom<IndexerDatabase, IndexerDatabase>;
+    api: z.ZodCustom<IndexerApi, IndexerApi>;
     localDb: z.ZodCustom<LocalDatabase, LocalDatabase>;
     config: z.ZodCustom<import("iso-conf").Conf<z.ZodObject<{
         privateKey: z.ZodOptional<z.ZodString>;
-        indexerMainnetUrl: z.ZodOptional<z.ZodURL>;
-        indexerCalibrationUrl: z.ZodOptional<z.ZodURL>;
+        apiUrl: z.ZodOptional<z.ZodURL>;
         chainId: z.ZodOptional<z.ZodNumber>;
         dbPath: z.ZodOptional<z.ZodString>;
         source: z.ZodOptional<z.ZodString>;
     }, z.core.$strip>>, import("iso-conf").Conf<z.ZodObject<{
         privateKey: z.ZodOptional<z.ZodString>;
-        indexerMainnetUrl: z.ZodOptional<z.ZodURL>;
-        indexerCalibrationUrl: z.ZodOptional<z.ZodURL>;
+        apiUrl: z.ZodOptional<z.ZodURL>;
         chainId: z.ZodOptional<z.ZodNumber>;
         dbPath: z.ZodOptional<z.ZodString>;
         source: z.ZodOptional<z.ZodString>;

package/dist/src/middleware.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"middleware.d.ts","sourceRoot":"","sources":["../../src/middleware.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,4BAA4B,CAAA;AAEvD,OAAO,EAAc,CAAC,EAAE,MAAM,OAAO,CAAA;AACrC,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,MAAM,CAAA;AAEtD,OAAO,KAAK,EAAE,eAAe,EAAE,aAAa,EAAE,MAAM,YAAY,CAAA;AAGhE,eAAO,MAAM,aAAa;;;;;;;;;;;;;;;;;;;;;;iBAQxB,CAAA;AAEF,eAAO,MAAM,iBAAiB;;;;;;;;;;;;;;;;;;;;;;6BA4B5B,CAAA"}
+{"version":3,"file":"middleware.d.ts","sourceRoot":"","sources":["../../src/middleware.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,4BAA4B,CAAA;AACvD,OAAO,EAAc,CAAC,EAAE,MAAM,OAAO,CAAA;AACrC,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,MAAM,CAAA;AAEtD,OAAO,KAAK,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,YAAY,CAAA;AAG3D,eAAO,MAAM,aAAa;;;;;;;;;;;;;;;;;;;;iBAQxB,CAAA;AAEF,eAAO,MAAM,iBAAiB;;;;;;;;;;;;;;;;;;;;6BAyB5B,CAAA"}

package/dist/src/middleware.js

@@ -1,9 +1,8 @@
-import { drizzle as drizzlePostgres } from 'drizzle-orm/node-postgres';
 import { middleware, z } from 'incur';
-import * as indexerSchema from "./indexer-schema.js";
+import { createIndexerApi } from "./api.js";
 import { config, createLocalDatabase, getClient } from "./utils.js";
 export const contextSchema = z.object({
-    indexerDb: z.custom(),
+    api: z.custom(),
     localDb: z.custom(),
     config: z.custom(),
     client: z.custom(),
@@ -12,8 +11,8 @@ export const contextSchema = z.object({
     isInteractive: z.boolean(),
 });
 export const contextMiddleware = middleware(async (c, next) => {
-    const { dbPath, chainId, indexerMainnetUrl, indexerCalibrationUrl, source } = config.store;
-    if (!dbPath || !chainId || !indexerMainnetUrl || !indexerCalibrationUrl || !source) {
+    const { dbPath, chainId, apiUrl, source } = config.store;
+    if (!dbPath || !chainId || !source) {
         return c.error({
             code: 'CONFIG_NOT_SET',
             message: 'Config not set. Please run `repair setup` first.',
@@ -21,12 +20,10 @@ export const contextMiddleware = middleware(async (c, next) => {
         });
     }
     const localDb = await createLocalDatabase(dbPath);
-    const indexerDb = drizzlePostgres(chainId === 314 ? indexerMainnetUrl : indexerCalibrationUrl, {
-        schema: indexerSchema,
-    });
+    const api = createIndexerApi({ baseUrl: apiUrl, chainId });
     const { client, chain } = getClient(chainId);
     c.set('localDb', localDb);
-    c.set('indexerDb', indexerDb);
+    c.set('api', api);
     c.set('config', config);
     c.set('client', client);
     c.set('chain', chain);
@@ -34,6 +31,5 @@ export const contextMiddleware = middleware(async (c, next) => {
     c.set('isInteractive', !c.agent && !c.formatExplicit);
     await next();
     localDb.$client.close();
-    await indexerDb.$client.end();
 });
 //# sourceMappingURL=middleware.js.map

package/dist/src/middleware.js.map

@@ -1 +1 @@
-{"version":3,"file":"middleware.js","sourceRoot":"","sources":["../../src/middleware.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,OAAO,IAAI,eAAe,EAAE,MAAM,2BAA2B,CAAA;AACtE,OAAO,EAAE,UAAU,EAAE,CAAC,EAAE,MAAM,OAAO,CAAA;AAErC,OAAO,KAAK,aAAa,MAAM,qBAAqB,CAAA;AAEpD,OAAO,EAAE,MAAM,EAAE,mBAAmB,EAAE,SAAS,EAAE,MAAM,YAAY,CAAA;AAEnE,MAAM,CAAC,MAAM,aAAa,GAAG,CAAC,CAAC,MAAM,CAAC;IACpC,SAAS,EAAE,CAAC,CAAC,MAAM,EAAmB;IACtC,OAAO,EAAE,CAAC,CAAC,MAAM,EAAiB;IAClC,MAAM,EAAE,CAAC,CAAC,MAAM,EAAiB;IACjC,MAAM,EAAE,CAAC,CAAC,MAAM,EAAqC;IACrD,KAAK,EAAE,CAAC,CAAC,MAAM,EAAS;IACxB,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE;IAClB,aAAa,EAAE,CAAC,CAAC,OAAO,EAAE;CAC3B,CAAC,CAAA;AAEF,MAAM,CAAC,MAAM,iBAAiB,GAAG,UAAU,CAAuB,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,EAAE;IAClF,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,iBAAiB,EAAE,qBAAqB,EAAE,MAAM,EAAE,GAAG,MAAM,CAAC,KAAK,CAAA;IAE1F,IAAI,CAAC,MAAM,IAAI,CAAC,OAAO,IAAI,CAAC,iBAAiB,IAAI,CAAC,qBAAqB,IAAI,CAAC,MAAM,EAAE,CAAC;QACnF,OAAO,CAAC,CAAC,KAAK,CAAC;YACb,IAAI,EAAE,gBAAgB;YACtB,OAAO,EAAE,kDAAkD;YAC3D,SAAS,EAAE,KAAK;SACjB,CAAC,CAAA;IACJ,CAAC;IAED,MAAM,OAAO,GAAG,MAAM,mBAAmB,CAAC,MAAM,CAAC,CAAA;IACjD,MAAM,SAAS,GAAG,eAAe,CAAC,OAAO,KAAK,GAAG,CAAC,CAAC,CAAC,iBAAiB,CAAC,CAAC,CAAC,qBAAqB,EAAE;QAC7F,MAAM,EAAE,aAAa;KACtB,CAAC,CAAA;IAEF,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG,SAAS,CAAC,OAAO,CAAC,CAAA;IAC5C,CAAC,CAAC,GAAG,CAAC,SAAS,EAAE,OAAO,CAAC,CAAA;IACzB,CAAC,CAAC,GAAG,CAAC,WAAW,EAAE,SAAS,CAAC,CAAA;IAC7B,CAAC,CAAC,GAAG,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAA;IACvB,CAAC,CAAC,GAAG,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAA;IACvB,CAAC,CAAC,GAAG,CAAC,OAAO,EAAE,KAAK,CAAC,CAAA;IACrB,CAAC,CAAC,GAAG,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAA;IACvB,CAAC,CAAC,GAAG,CAAC,eAAe,EAAE,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC,cAAc,CAAC,CAAA;IACrD,MAAM,IAAI,EAAE,CAAA;IAEZ,OAAO,CAAC,OAAO,CAAC,KAAK,EAAE,CAAA;IACvB,MAAM,SAAS,CAAC,OAAO,CAAC,GAAG,EAAE,CAAA;AAC/B,CAAC,CAAC,CAAA"}
+{"version":3,"file":"middleware.js","sourceRoot":"","sources":["../../src/middleware.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,UAAU,EAAE,CAAC,EAAE,MAAM,OAAO,CAAA;AAErC,OAAO,EAAE,gBAAgB,EAAE,MAAM,UAAU,CAAA;AAE3C,OAAO,EAAE,MAAM,EAAE,mBAAmB,EAAE,SAAS,EAAE,MAAM,YAAY,CAAA;AAEnE,MAAM,CAAC,MAAM,aAAa,GAAG,CAAC,CAAC,MAAM,CAAC;IACpC,GAAG,EAAE,CAAC,CAAC,MAAM,EAAc;IAC3B,OAAO,EAAE,CAAC,CAAC,MAAM,EAAiB;IAClC,MAAM,EAAE,CAAC,CAAC,MAAM,EAAiB;IACjC,MAAM,EAAE,CAAC,CAAC,MAAM,EAAqC;IACrD,KAAK,EAAE,CAAC,CAAC,MAAM,EAAS;IACxB,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE;IAClB,aAAa,EAAE,CAAC,CAAC,OAAO,EAAE;CAC3B,CAAC,CAAA;AAEF,MAAM,CAAC,MAAM,iBAAiB,GAAG,UAAU,CAAuB,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,EAAE;IAClF,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,CAAC,KAAK,CAAA;IAExD,IAAI,CAAC,MAAM,IAAI,CAAC,OAAO,IAAI,CAAC,MAAM,EAAE,CAAC;QACnC,OAAO,CAAC,CAAC,KAAK,CAAC;YACb,IAAI,EAAE,gBAAgB;YACtB,OAAO,EAAE,kDAAkD;YAC3D,SAAS,EAAE,KAAK;SACjB,CAAC,CAAA;IACJ,CAAC;IAED,MAAM,OAAO,GAAG,MAAM,mBAAmB,CAAC,MAAM,CAAC,CAAA;IACjD,MAAM,GAAG,GAAG,gBAAgB,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC,CAAA;IAE1D,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG,SAAS,CAAC,OAAO,CAAC,CAAA;IAC5C,CAAC,CAAC,GAAG,CAAC,SAAS,EAAE,OAAO,CAAC,CAAA;IACzB,CAAC,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,CAAC,CAAA;IACjB,CAAC,CAAC,GAAG,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAA;IACvB,CAAC,CAAC,GAAG,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAA;IACvB,CAAC,CAAC,GAAG,CAAC,OAAO,EAAE,KAAK,CAAC,CAAA;IACrB,CAAC,CAAC,GAAG,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAA;IACvB,CAAC,CAAC,GAAG,CAAC,eAAe,EAAE,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC,cAAc,CAAC,CAAA;IACrD,MAAM,IAAI,EAAE,CAAA;IAEZ,OAAO,CAAC,OAAO,CAAC,KAAK,EAAE,CAAA;AACzB,CAAC,CAAC,CAAA"}

package/dist/src/pipeline/add-pieces.d.ts

@@ -1,12 +1,12 @@
 import type { RepairSelect } from '../local-schema.ts';
-import type { IndexerDatabase, LocalDatabase, WalletClient } from '../types.ts';
+import type { IndexerApi, LocalDatabase, WalletClient } from '../types.ts';
 export type RunPullPiecesPhaseOptions = {
     localDb: LocalDatabase;
-    indexerDb: IndexerDatabase;
+    api: IndexerApi;
     repair: RepairSelect;
     concurrency: number;
     batchSize: number;
     client: WalletClient;
 };
-export declare function runAddPieces({ localDb, indexerDb, repair, concurrency, batchSize, client, }: RunPullPiecesPhaseOptions): Promise<void>;
+export declare function runAddPieces({ localDb, api, repair, concurrency, batchSize, client, }: RunPullPiecesPhaseOptions): Promise<void>;
 //# sourceMappingURL=add-pieces.d.ts.map

package/dist/src/pipeline/add-pieces.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"add-pieces.d.ts","sourceRoot":"","sources":["../../../src/pipeline/add-pieces.ts"],"names":[],"mappings":"AASA,OAAO,KAAK,EAAmB,YAAY,EAAE,MAAM,oBAAoB,CAAA;AACvE,OAAO,KAAK,EAAE,eAAe,EAAE,aAAa,EAAE,YAAY,EAAE,MAAM,aAAa,CAAA;AAG/E,MAAM,MAAM,yBAAyB,GAAG;IACtC,OAAO,EAAE,aAAa,CAAA;IACtB,SAAS,EAAE,eAAe,CAAA;IAC1B,MAAM,EAAE,YAAY,CAAA;IACpB,WAAW,EAAE,MAAM,CAAA;IACnB,SAAS,EAAE,MAAM,CAAA;IACjB,MAAM,EAAE,YAAY,CAAA;CACrB,CAAA;AA0ID,wBAAsB,YAAY,CAAC,EACjC,OAAO,EACP,SAAS,EACT,MAAM,EACN,WAAW,EACX,SAAS,EACT,MAAM,GACP,EAAE,yBAAyB,GAAG,OAAO,CAAC,IAAI,CAAC,CAiE3C"}
+{"version":3,"file":"add-pieces.d.ts","sourceRoot":"","sources":["../../../src/pipeline/add-pieces.ts"],"names":[],"mappings":"AAUA,OAAO,KAAK,EAAmB,YAAY,EAAE,MAAM,oBAAoB,CAAA;AACvE,OAAO,KAAK,EAAE,UAAU,EAAE,aAAa,EAAE,YAAY,EAAE,MAAM,aAAa,CAAA;AAG1E,MAAM,MAAM,yBAAyB,GAAG;IACtC,OAAO,EAAE,aAAa,CAAA;IACtB,GAAG,EAAE,UAAU,CAAA;IACf,MAAM,EAAE,YAAY,CAAA;IACpB,WAAW,EAAE,MAAM,CAAA;IACnB,SAAS,EAAE,MAAM,CAAA;IACjB,MAAM,EAAE,YAAY,CAAA;CACrB,CAAA;AA2ID,wBAAsB,YAAY,CAAC,EACjC,OAAO,EACP,GAAG,EACH,MAAM,EACN,WAAW,EACX,SAAS,EACT,MAAM,GACP,EAAE,yBAAyB,GAAG,OAAO,CAAC,IAAI,CAAC,CAyE3C"}

package/dist/src/pipeline/add-pieces.js

@@ -3,22 +3,22 @@ import * as Piece from '@filoz/synapse-core/piece';
 import * as SP from '@filoz/synapse-core/sp';
 import { and, asc, eq, gt, inArray } from 'drizzle-orm';
 import PQueue from 'p-queue';
+import { listDataSetPieces } from "../api.js";
 import { dedupeCids } from "../db/dedupe-cids.js";
 import { getTargetDataset } from "../db/get-target-dataset.js";
 import { repairUpdate } from "../db/repair-update.js";
 import { upsertOperations } from "../db/upsert-operations.js";
 import { completeConfirmedOperations, excludeOperationsByCid, hashLink, operationsToPullPieces } from "../utils.js";
-function createAddPiecesWorker({ localDb, indexerDb, repair, client, state, log }) {
+function createAddPiecesWorker({ localDb, targetCids, repair, client, state, log }) {
     return async (options) => {
         let completedOps = 0;
         let failedOps = 0;
         let operations = options.operations;
-        const isRepair = repair.repairDataSetId == null;
         const group = log.group(`Batch ${options.batchNumber}/${state.totalBatches}`);
         try {
             const dataset = await getTargetDataset({ localDb, repairId: repair.id, client });
-            if (isRepair) {
-                operations = await dedupeCids({ indexerDb, localDb, dataSetId: dataset.dataSetId, operations });
+            if (targetCids) {
+                operations = await dedupeCids({ localDb, existingCids: targetCids, operations });
             }
             const operationsBeforeConfirmationCheck = operations.length;
             operations = await completeConfirmedOperations({ localDb, client, operations });
@@ -67,7 +67,7 @@ function createAddPiecesWorker({ localDb, indexerDb, repair, client, state, log
                     clientDataSetId: dataset.clientDataSetId,
                     pieces: operations.map((operation) => ({
                         pieceCid: Piece.from(operation.cid),
-                        metadata: isRepair ? undefined : operation.metadata,
+                        metadata: targetCids ? undefined : operation.metadata,
                     })),
                 });
                 await upsertOperations({
@@ -81,6 +81,8 @@ function createAddPiecesWorker({ localDb, indexerDb, repair, client, state, log
                 const waitForAddPieces = await SP.waitForAddPieces(addPiecesResult);
                 state.completedOperations += operations.length;
                 completedOps += operations.length;
+                for (const operation of operations)
+                    targetCids?.add(operation.cid);
                 await upsertOperations({
                     localDb,
                     operations: operations.map((operation) => ({
@@ -108,7 +110,7 @@ function createAddPiecesWorker({ localDb, indexerDb, repair, client, state, log
         }
     };
 }
-export async function runAddPieces({ localDb, indexerDb, repair, concurrency, batchSize, client, }) {
+export async function runAddPieces({ localDb, api, repair, concurrency, batchSize, client, }) {
     const localSchema = localDb._.fullSchema;
     let cursor = 0;
     const totalOperations = await localDb.$count(localSchema.operations, and(eq(localSchema.operations.repairId, repair.id), eq(localSchema.operations.type, 'add_piece'), inArray(localSchema.operations.status, ['pending', 'failed'])));
@@ -123,6 +125,12 @@ export async function runAddPieces({ localDb, indexerDb, repair, concurrency, ba
         title: 'Adding pieces',
         limit: 1,
     });
+    let targetCids;
+    if (repair.repairDataSetId == null && totalOperations > 0) {
+        const dataset = await getTargetDataset({ localDb, repairId: repair.id, client });
+        const pieces = await listDataSetPieces(api, dataset.dataSetId, { removed: false });
+        targetCids = new Set(pieces.flatMap((piece) => (piece.cid == null ? [] : [piece.cid])));
+    }
     async function getNextBatch() {
         const operations = await localDb.query.operations.findMany({
             where: and(eq(localSchema.operations.repairId, repair.id), eq(localSchema.operations.type, 'add_piece'), inArray(localSchema.operations.status, ['pending', 'failed']), gt(localSchema.operations.id, cursor)),
@@ -135,7 +143,7 @@ export async function runAddPieces({ localDb, indexerDb, repair, concurrency, ba
         cursor = operations.at(-1)?.id ?? cursor;
         return operations;
     }
-    const addPiecesJob = createAddPiecesWorker({ localDb, indexerDb, repair, client, state, log });
+    const addPiecesJob = createAddPiecesWorker({ localDb, targetCids, repair, client, state, log });
     const queue = new PQueue({ concurrency });
     while (true) {
         await queue.onSizeLessThan(concurrency);

package/dist/src/pipeline/add-pieces.js.map

@@ -1 +1 @@
-{"version":3,"file":"add-pieces.js","sourceRoot":"","sources":["../../../src/pipeline/add-pieces.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,gBAAgB,CAAA;AACxC,OAAO,KAAK,KAAK,MAAM,2BAA2B,CAAA;AAClD,OAAO,KAAK,EAAE,MAAM,wBAAwB,CAAA;AAC5C,OAAO,EAAE,GAAG,EAAE,GAAG,EAAE,EAAE,EAAE,EAAE,EAAE,OAAO,EAAE,MAAM,aAAa,CAAA;AACvD,OAAO,MAAM,MAAM,SAAS,CAAA;AAC5B,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAA;AACjD,OAAO,EAAE,gBAAgB,EAAE,MAAM,6BAA6B,CAAA;AAC9D,OAAO,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAA;AACrD,OAAO,EAAE,gBAAgB,EAAE,MAAM,4BAA4B,CAAA;AAG7D,OAAO,EAAE,2BAA2B,EAAE,sBAAsB,EAAE,QAAQ,EAAE,sBAAsB,EAAE,MAAM,aAAa,CAAA;AAiCnH,SAAS,qBAAqB,CAAC,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG,EAAgC;IAC7G,OAAO,KAAK,EAAE,OAA4B,EAAE,EAAE;QAC5C,IAAI,YAAY,GAAG,CAAC,CAAA;QACpB,IAAI,SAAS,GAAG,CAAC,CAAA;QACjB,IAAI,UAAU,GAAsB,OAAO,CAAC,UAAU,CAAA;QACtD,MAAM,QAAQ,GAAG,MAAM,CAAC,eAAe,IAAI,IAAI,CAAA;QAE/C,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC,SAAS,OAAO,CAAC,WAAW,IAAI,KAAK,CAAC,YAAY,EAAE,CAAC,CAAA;QAE7E,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,MAAM,gBAAgB,CAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,CAAC,EAAE,EAAE,MAAM,EAAE,CAAC,CAAA;YAGhF,IAAI,QAAQ,EAAE,CAAC;gBACb,UAAU,GAAG,MAAM,UAAU,CAAC,EAAE,SAAS,EAAE,OAAO,EAAE,SAAS,EAAE,OAAO,CAAC,SAAS,EAAE,UAAU,EAAE,CAAC,CAAA;YACjG,CAAC;YAED,MAAM,iCAAiC,GAAG,UAAU,CAAC,MAAM,CAAA;YAC3D,UAAU,GAAG,MAAM,2BAA2B,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC,CAAA;YAC/E,MAAM,mBAAmB,GAAG,iCAAiC,GAAG,UAAU,CAAC,MAAM,CAAA;YACjF,IAAI,mBAAmB,GAAG,CAAC,EAAE,CAAC;gBAC5B,KAAK,CAAC,mBAAmB,IAAI,mBAAmB,CAAA;gBAChD,YAAY,IAAI,mBAAmB,CAAA;YACrC,CAAC;YAED,KAAK,CAAC,OAAO,CAAC,WAAW,UAAU,CAAC,MAAM,YAAY,CAAC,CAAA;YAEvD,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC1B,MAAM,UAAU,GAAG,MAAM,EAAE,CAAC,iBAAiB,CAAC,MAAM,EAAE;oBACpD,UAAU,EAAE,MAAM,CAAC,iBAAiB;oBACpC,SAAS,EAAE,OAAO,CAAC,SAAS;oBAC5B,eAAe,EAAE,OAAO,CAAC,eAAe;oBACxC,MAAM,EAAE,sBAAsB,CAAC,UAAU,CAAC;oBAC1C,OAAO,EAAE,IAAI,GAAG,EAAE,GAAG,EAAE;oBACvB,QAAQ,EAAE,CAAC,MAAM,EAAE,EAAE;wBACnB,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,KAAK,UAAU,CAAC,CAAC,MAAM,CAAA;wBACrF,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,MAAM,CAAA;wBAChF,KAAK,CAAC,OAAO,CAAC,QAAQ,SAAS,eAAe,MAAM,SAAS,CAAC,CAAA;oBAChE,CAAC;iBACF,CAAC,CAAA;gBAEF,KAAK,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,UAAU,CAAC,MAAM,EAAE,CAAC;oBACrD,MAAM,GAAG,GAAG,QAAQ,CAAC,QAAQ,EAAE,CAAA;oBAC/B,IAAI,MAAM,KAAK,UAAU,EAAE,CAAC;wBAC1B,KAAK,CAAC,gBAAgB,EAAE,CAAA;wBACxB,SAAS,EAAE,CAAA;wBACX,MAAM,EAAE,iBAAiB,EAAE,iBAAiB,EAAE,GAAG,sBAAsB,CAAC,UAAU,EAAE,GAAG,CAAC,CAAA;wBACxF,UAAU,GAAG,iBAAiB,CAAA;wBAC9B,MAAM,gBAAgB,CAAC;4BACrB,OAAO;4BACP,UAAU,EAAE,iBAAiB,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;gCAChD,GAAG,SAAS;gCACZ,MAAM,EAAE,QAAQ;gCAChB,KAAK,EAAE,2BAA2B,MAAM,EAAE;6BAC3C,CAAC,CAAC;yBACJ,CAAC,CAAA;oBACJ,CAAC;gBACH,CAAC;YACH,CAAC;YAGD,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC1B,KAAK,CAAC,OAAO,CAAC,UAAU,UAAU,CAAC,MAAM,YAAY,CAAC,CAAA;gBACtD,MAAM,eAAe,GAAG,MAAM,EAAE,CAAC,SAAS,CAAC,MAAM,EAAE;oBACjD,UAAU,EAAE,MAAM,CAAC,iBAAiB;oBACpC,SAAS,EAAE,OAAO,CAAC,SAAS;oBAC5B,eAAe,EAAE,OAAO,CAAC,eAAe;oBACxC,MAAM,EAAE,UAAU,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;wBACrC,QAAQ,EAAE,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC;wBACnC,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,QAAQ;qBACpD,CAAC,CAAC;iBACJ,CAAC,CAAA;gBACF,MAAM,gBAAgB,CAAC;oBACrB,OAAO;oBACP,UAAU,EAAE,UAAU,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;wBACzC,GAAG,SAAS;wBACZ,MAAM,EAAE,eAAe,CAAC,MAAM;qBAC/B,CAAC,CAAC;iBACJ,CAAC,CAAA;gBAEF,KAAK,CAAC,OAAO,CAAC,0BAA0B,QAAQ,CAAC,eAAe,CAAC,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAA;gBAC5F,MAAM,gBAAgB,GAAG,MAAM,EAAE,CAAC,gBAAgB,CAAC,eAAe,CAAC,CAAA;gBACnE,KAAK,CAAC,mBAAmB,IAAI,UAAU,CAAC,MAAM,CAAA;gBAC9C,YAAY,IAAI,UAAU,CAAC,MAAM,CAAA;gBACjC,MAAM,gBAAgB,CAAC;oBACrB,OAAO;oBACP,UAAU,EAAE,UAAU,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;wBACzC,GAAG,SAAS;wBACZ,MAAM,EAAE,WAAW;wBACnB,KAAK,EAAE,IAAI;wBACX,MAAM,EAAE,gBAAgB,CAAC,MAAM;qBAChC,CAAC,CAAC;iBACJ,CAAC,CAAA;YACJ,CAAC;YACD,KAAK,CAAC,OAAO,CAAC,SAAS,YAAY,WAAW,SAAS,SAAS,CAAC,CAAA;QACnE,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,KAAK,CAAC,gBAAgB,IAAI,UAAU,CAAC,MAAM,CAAA;YAC3C,MAAM,OAAO,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,CAAA;YACxE,KAAK,CAAC,OAAO,CAAC,GAAG,OAAO,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,EAAE,CAAC,CAAA;YAC/C,MAAM,gBAAgB,CAAC;gBACrB,OAAO;gBACP,UAAU,EAAE,UAAU,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;oBACzC,GAAG,SAAS;oBACZ,MAAM,EAAE,QAAQ;oBAChB,KAAK,EAAE,OAAO;iBACf,CAAC,CAAC;aACJ,CAAC,CAAA;QACJ,CAAC;IACH,CAAC,CAAA;AACH,CAAC;AAKD,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,EACjC,OAAO,EACP,SAAS,EACT,MAAM,EACN,WAAW,EACX,SAAS,EACT,MAAM,GACoB;IAC1B,MAAM,WAAW,GAAG,OAAO,CAAC,CAAC,CAAC,UAAU,CAAA;IACxC,IAAI,MAAM,GAAG,CAAC,CAAA;IAEd,MAAM,eAAe,GAAG,MAAM,OAAO,CAAC,MAAM,CAC1C,WAAW,CAAC,UAAU,EACtB,GAAG,CACD,EAAE,CAAC,WAAW,CAAC,UAAU,CAAC,QAAQ,EAAE,MAAM,CAAC,EAAE,CAAC,EAC9C,EAAE,CAAC,WAAW,CAAC,UAAU,CAAC,IAAI,EAAE,WAAW,CAAC,EAC5C,OAAO,CAAC,WAAW,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC,CAC9D,CACF,CAAA;IACD,IAAI,WAAW,GAAG,CAAC,CAAA;IACnB,MAAM,KAAK,GAAG;QACZ,YAAY,EAAE,IAAI,CAAC,IAAI,CAAC,eAAe,GAAG,SAAS,CAAC;QACpD,eAAe;QACf,mBAAmB,EAAE,CAAC;QACtB,gBAAgB,EAAE,CAAC;KACpB,CAAA;IAED,MAAM,GAAG,GAAG,OAAO,CAAC;QAClB,KAAK,EAAE,eAAe;QACtB,KAAK,EAAE,CAAC;KACT,CAAC,CAAA;IAEF,KAAK,UAAU,YAAY;QACzB,MAAM,UAAU,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,UAAU,CAAC,QAAQ,CAAC;YACzD,KAAK,EAAE,GAAG,CACR,EAAE,CAAC,WAAW,CAAC,UAAU,CAAC,QAAQ,EAAE,MAAM,CAAC,EAAE,CAAC,EAC9C,EAAE,CAAC,WAAW,CAAC,UAAU,CAAC,IAAI,EAAE,WAAW,CAAC,EAC5C,OAAO,CAAC,WAAW,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC,EAC7D,EAAE,CAAC,WAAW,CAAC,UAAU,CAAC,EAAE,EAAE,MAAM,CAAC,CACtC;YACD,OAAO,EAAE,CAAC,GAAG,CAAC,WAAW,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;YACzC,KAAK,EAAE,SAAS;SACjB,CAAC,CAAA;QACF,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC5B,OAAO,IAAI,CAAA;QACb,CAAC;QAED,MAAM,GAAG,UAAU,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,IAAI,MAAM,CAAA;QACxC,OAAO,UAAU,CAAA;IACnB,CAAC;IAED,MAAM,YAAY,GAAG,qBAAqB,CAAC,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,CAAA;IAC9F,MAAM,KAAK,GAAG,IAAI,MAAM,CAAC,EAAE,WAAW,EAAE,CAAC,CAAA;IAEzC,OAAO,IAAI,EAAE,CAAC;QACZ,MAAM,KAAK,CAAC,cAAc,CAAC,WAAW,CAAC,CAAA;QACvC,MAAM,UAAU,GAAG,MAAM,YAAY,EAAE,CAAA;QACvC,IAAI,CAAC,UAAU;YAAE,MAAK;QACtB,WAAW,EAAE,CAAA;QACb,MAAM,kBAAkB,GAAG,WAAW,CAAA;QACtC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,YAAY,CAAC,EAAE,UAAU,EAAE,WAAW,EAAE,kBAAkB,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;IACrG,CAAC;IAED,MAAM,KAAK,CAAC,MAAM,EAAE,CAAA;IAEpB,GAAG,CAAC,OAAO,CAAC,SAAS,KAAK,CAAC,mBAAmB,YAAY,KAAK,CAAC,gBAAgB,SAAS,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAA;IAE7G,MAAM,YAAY,CAAC;QACjB,OAAO;QACP,QAAQ,EAAE,MAAM,CAAC,EAAE;QACnB,MAAM,EAAE,KAAK,CAAC,gBAAgB,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,WAAW;KAC5D,CAAC,CAAA;AACJ,CAAC"}
+{"version":3,"file":"add-pieces.js","sourceRoot":"","sources":["../../../src/pipeline/add-pieces.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,gBAAgB,CAAA;AACxC,OAAO,KAAK,KAAK,MAAM,2BAA2B,CAAA;AAClD,OAAO,KAAK,EAAE,MAAM,wBAAwB,CAAA;AAC5C,OAAO,EAAE,GAAG,EAAE,GAAG,EAAE,EAAE,EAAE,EAAE,EAAE,OAAO,EAAE,MAAM,aAAa,CAAA;AACvD,OAAO,MAAM,MAAM,SAAS,CAAA;AAC5B,OAAO,EAAE,iBAAiB,EAAE,MAAM,WAAW,CAAA;AAC7C,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAA;AACjD,OAAO,EAAE,gBAAgB,EAAE,MAAM,6BAA6B,CAAA;AAC9D,OAAO,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAA;AACrD,OAAO,EAAE,gBAAgB,EAAE,MAAM,4BAA4B,CAAA;AAG7D,OAAO,EAAE,2BAA2B,EAAE,sBAAsB,EAAE,QAAQ,EAAE,sBAAsB,EAAE,MAAM,aAAa,CAAA;AAkCnH,SAAS,qBAAqB,CAAC,EAAE,OAAO,EAAE,UAAU,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG,EAAgC;IAC9G,OAAO,KAAK,EAAE,OAA4B,EAAE,EAAE;QAC5C,IAAI,YAAY,GAAG,CAAC,CAAA;QACpB,IAAI,SAAS,GAAG,CAAC,CAAA;QACjB,IAAI,UAAU,GAAsB,OAAO,CAAC,UAAU,CAAA;QAEtD,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC,SAAS,OAAO,CAAC,WAAW,IAAI,KAAK,CAAC,YAAY,EAAE,CAAC,CAAA;QAE7E,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,MAAM,gBAAgB,CAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,CAAC,EAAE,EAAE,MAAM,EAAE,CAAC,CAAA;YAGhF,IAAI,UAAU,EAAE,CAAC;gBACf,UAAU,GAAG,MAAM,UAAU,CAAC,EAAE,OAAO,EAAE,YAAY,EAAE,UAAU,EAAE,UAAU,EAAE,CAAC,CAAA;YAClF,CAAC;YAED,MAAM,iCAAiC,GAAG,UAAU,CAAC,MAAM,CAAA;YAC3D,UAAU,GAAG,MAAM,2BAA2B,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC,CAAA;YAC/E,MAAM,mBAAmB,GAAG,iCAAiC,GAAG,UAAU,CAAC,MAAM,CAAA;YACjF,IAAI,mBAAmB,GAAG,CAAC,EAAE,CAAC;gBAC5B,KAAK,CAAC,mBAAmB,IAAI,mBAAmB,CAAA;gBAChD,YAAY,IAAI,mBAAmB,CAAA;YACrC,CAAC;YAED,KAAK,CAAC,OAAO,CAAC,WAAW,UAAU,CAAC,MAAM,YAAY,CAAC,CAAA;YAEvD,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC1B,MAAM,UAAU,GAAG,MAAM,EAAE,CAAC,iBAAiB,CAAC,MAAM,EAAE;oBACpD,UAAU,EAAE,MAAM,CAAC,iBAAiB;oBACpC,SAAS,EAAE,OAAO,CAAC,SAAS;oBAC5B,eAAe,EAAE,OAAO,CAAC,eAAe;oBACxC,MAAM,EAAE,sBAAsB,CAAC,UAAU,CAAC;oBAC1C,OAAO,EAAE,IAAI,GAAG,EAAE,GAAG,EAAE;oBACvB,QAAQ,EAAE,CAAC,MAAM,EAAE,EAAE;wBACnB,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,KAAK,UAAU,CAAC,CAAC,MAAM,CAAA;wBACrF,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,MAAM,CAAA;wBAChF,KAAK,CAAC,OAAO,CAAC,QAAQ,SAAS,eAAe,MAAM,SAAS,CAAC,CAAA;oBAChE,CAAC;iBACF,CAAC,CAAA;gBAEF,KAAK,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,UAAU,CAAC,MAAM,EAAE,CAAC;oBACrD,MAAM,GAAG,GAAG,QAAQ,CAAC,QAAQ,EAAE,CAAA;oBAC/B,IAAI,MAAM,KAAK,UAAU,EAAE,CAAC;wBAC1B,KAAK,CAAC,gBAAgB,EAAE,CAAA;wBACxB,SAAS,EAAE,CAAA;wBACX,MAAM,EAAE,iBAAiB,EAAE,iBAAiB,EAAE,GAAG,sBAAsB,CAAC,UAAU,EAAE,GAAG,CAAC,CAAA;wBACxF,UAAU,GAAG,iBAAiB,CAAA;wBAC9B,MAAM,gBAAgB,CAAC;4BACrB,OAAO;4BACP,UAAU,EAAE,iBAAiB,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;gCAChD,GAAG,SAAS;gCACZ,MAAM,EAAE,QAAQ;gCAChB,KAAK,EAAE,2BAA2B,MAAM,EAAE;6BAC3C,CAAC,CAAC;yBACJ,CAAC,CAAA;oBACJ,CAAC;gBACH,CAAC;YACH,CAAC;YAGD,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC1B,KAAK,CAAC,OAAO,CAAC,UAAU,UAAU,CAAC,MAAM,YAAY,CAAC,CAAA;gBACtD,MAAM,eAAe,GAAG,MAAM,EAAE,CAAC,SAAS,CAAC,MAAM,EAAE;oBACjD,UAAU,EAAE,MAAM,CAAC,iBAAiB;oBACpC,SAAS,EAAE,OAAO,CAAC,SAAS;oBAC5B,eAAe,EAAE,OAAO,CAAC,eAAe;oBACxC,MAAM,EAAE,UAAU,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;wBACrC,QAAQ,EAAE,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC;wBACnC,QAAQ,EAAE,UAAU,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,QAAQ;qBACtD,CAAC,CAAC;iBACJ,CAAC,CAAA;gBACF,MAAM,gBAAgB,CAAC;oBACrB,OAAO;oBACP,UAAU,EAAE,UAAU,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;wBACzC,GAAG,SAAS;wBACZ,MAAM,EAAE,eAAe,CAAC,MAAM;qBAC/B,CAAC,CAAC;iBACJ,CAAC,CAAA;gBAEF,KAAK,CAAC,OAAO,CAAC,0BAA0B,QAAQ,CAAC,eAAe,CAAC,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAA;gBAC5F,MAAM,gBAAgB,GAAG,MAAM,EAAE,CAAC,gBAAgB,CAAC,eAAe,CAAC,CAAA;gBACnE,KAAK,CAAC,mBAAmB,IAAI,UAAU,CAAC,MAAM,CAAA;gBAC9C,YAAY,IAAI,UAAU,CAAC,MAAM,CAAA;gBACjC,KAAK,MAAM,SAAS,IAAI,UAAU;oBAAE,UAAU,EAAE,GAAG,CAAC,SAAS,CAAC,GAAG,CAAC,CAAA;gBAClE,MAAM,gBAAgB,CAAC;oBACrB,OAAO;oBACP,UAAU,EAAE,UAAU,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;wBACzC,GAAG,SAAS;wBACZ,MAAM,EAAE,WAAW;wBACnB,KAAK,EAAE,IAAI;wBACX,MAAM,EAAE,gBAAgB,CAAC,MAAM;qBAChC,CAAC,CAAC;iBACJ,CAAC,CAAA;YACJ,CAAC;YACD,KAAK,CAAC,OAAO,CAAC,SAAS,YAAY,WAAW,SAAS,SAAS,CAAC,CAAA;QACnE,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,KAAK,CAAC,gBAAgB,IAAI,UAAU,CAAC,MAAM,CAAA;YAC3C,MAAM,OAAO,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,CAAA;YACxE,KAAK,CAAC,OAAO,CAAC,GAAG,OAAO,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,EAAE,CAAC,CAAA;YAC/C,MAAM,gBAAgB,CAAC;gBACrB,OAAO;gBACP,UAAU,EAAE,UAAU,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;oBACzC,GAAG,SAAS;oBACZ,MAAM,EAAE,QAAQ;oBAChB,KAAK,EAAE,OAAO;iBACf,CAAC,CAAC;aACJ,CAAC,CAAA;QACJ,CAAC;IACH,CAAC,CAAA;AACH,CAAC;AAKD,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,EACjC,OAAO,EACP,GAAG,EACH,MAAM,EACN,WAAW,EACX,SAAS,EACT,MAAM,GACoB;IAC1B,MAAM,WAAW,GAAG,OAAO,CAAC,CAAC,CAAC,UAAU,CAAA;IACxC,IAAI,MAAM,GAAG,CAAC,CAAA;IAEd,MAAM,eAAe,GAAG,MAAM,OAAO,CAAC,MAAM,CAC1C,WAAW,CAAC,UAAU,EACtB,GAAG,CACD,EAAE,CAAC,WAAW,CAAC,UAAU,CAAC,QAAQ,EAAE,MAAM,CAAC,EAAE,CAAC,EAC9C,EAAE,CAAC,WAAW,CAAC,UAAU,CAAC,IAAI,EAAE,WAAW,CAAC,EAC5C,OAAO,CAAC,WAAW,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC,CAC9D,CACF,CAAA;IACD,IAAI,WAAW,GAAG,CAAC,CAAA;IACnB,MAAM,KAAK,GAAG;QACZ,YAAY,EAAE,IAAI,CAAC,IAAI,CAAC,eAAe,GAAG,SAAS,CAAC;QACpD,eAAe;QACf,mBAAmB,EAAE,CAAC;QACtB,gBAAgB,EAAE,CAAC;KACpB,CAAA;IAED,MAAM,GAAG,GAAG,OAAO,CAAC;QAClB,KAAK,EAAE,eAAe;QACtB,KAAK,EAAE,CAAC;KACT,CAAC,CAAA;IAGF,IAAI,UAAmC,CAAA;IACvC,IAAI,MAAM,CAAC,eAAe,IAAI,IAAI,IAAI,eAAe,GAAG,CAAC,EAAE,CAAC;QAC1D,MAAM,OAAO,GAAG,MAAM,gBAAgB,CAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,CAAC,EAAE,EAAE,MAAM,EAAE,CAAC,CAAA;QAChF,MAAM,MAAM,GAAG,MAAM,iBAAiB,CAAC,GAAG,EAAE,OAAO,CAAC,SAAS,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAA;QAClF,UAAU,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,IAAI,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAA;IACzF,CAAC;IAED,KAAK,UAAU,YAAY;QACzB,MAAM,UAAU,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,UAAU,CAAC,QAAQ,CAAC;YACzD,KAAK,EAAE,GAAG,CACR,EAAE,CAAC,WAAW,CAAC,UAAU,CAAC,QAAQ,EAAE,MAAM,CAAC,EAAE,CAAC,EAC9C,EAAE,CAAC,WAAW,CAAC,UAAU,CAAC,IAAI,EAAE,WAAW,CAAC,EAC5C,OAAO,CAAC,WAAW,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC,EAC7D,EAAE,CAAC,WAAW,CAAC,UAAU,CAAC,EAAE,EAAE,MAAM,CAAC,CACtC;YACD,OAAO,EAAE,CAAC,GAAG,CAAC,WAAW,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;YACzC,KAAK,EAAE,SAAS;SACjB,CAAC,CAAA;QACF,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC5B,OAAO,IAAI,CAAA;QACb,CAAC;QAED,MAAM,GAAG,UAAU,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,IAAI,MAAM,CAAA;QACxC,OAAO,UAAU,CAAA;IACnB,CAAC;IAED,MAAM,YAAY,GAAG,qBAAqB,CAAC,EAAE,OAAO,EAAE,UAAU,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,CAAA;IAC/F,MAAM,KAAK,GAAG,IAAI,MAAM,CAAC,EAAE,WAAW,EAAE,CAAC,CAAA;IAEzC,OAAO,IAAI,EAAE,CAAC;QACZ,MAAM,KAAK,CAAC,cAAc,CAAC,WAAW,CAAC,CAAA;QACvC,MAAM,UAAU,GAAG,MAAM,YAAY,EAAE,CAAA;QACvC,IAAI,CAAC,UAAU;YAAE,MAAK;QACtB,WAAW,EAAE,CAAA;QACb,MAAM,kBAAkB,GAAG,WAAW,CAAA;QACtC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,YAAY,CAAC,EAAE,UAAU,EAAE,WAAW,EAAE,kBAAkB,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;IACrG,CAAC;IAED,MAAM,KAAK,CAAC,MAAM,EAAE,CAAA;IAEpB,GAAG,CAAC,OAAO,CAAC,SAAS,KAAK,CAAC,mBAAmB,YAAY,KAAK,CAAC,gBAAgB,SAAS,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAA;IAE7G,MAAM,YAAY,CAAC;QACjB,OAAO;QACP,QAAQ,EAAE,MAAM,CAAC,EAAE;QACnB,MAAM,EAAE,KAAK,CAAC,gBAAgB,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,WAAW;KAC5D,CAAC,CAAA;AACJ,CAAC"}

package/dist/src/pipeline/create-datasets.d.ts

@@ -1,14 +1,14 @@
 import type { Address } from 'viem';
 import type { RepairSelect } from '../local-schema.ts';
-import type { IndexerDatabase, LocalDatabase, WalletClient } from '../types.ts';
+import type { IndexerApi, LocalDatabase, WalletClient } from '../types.ts';
 export type EnsureRepairDatasetOptions = {
     source: string;
     localDb: LocalDatabase;
-    indexerDb: IndexerDatabase;
+    api: IndexerApi;
     client: WalletClient;
     repair: RepairSelect;
     payer: Address;
 };
-export declare function ensureRepairDataset({ source, localDb, indexerDb, client, repair, payer, }: EnsureRepairDatasetOptions): Promise<bigint>;
-export declare function ensureReplicateDataset({ localDb, indexerDb, client, repair, payer, }: EnsureRepairDatasetOptions): Promise<bigint>;
+export declare function ensureRepairDataset({ source, localDb, api, client, repair, payer }: EnsureRepairDatasetOptions): Promise<bigint>;
+export declare function ensureReplicateDataset({ localDb, api, client, repair, payer }: EnsureRepairDatasetOptions): Promise<bigint>;
 //# sourceMappingURL=create-datasets.d.ts.map

package/dist/src/pipeline/create-datasets.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"create-datasets.d.ts","sourceRoot":"","sources":["../../../src/pipeline/create-datasets.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,MAAM,CAAA;AAGnC,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAA;AACtD,OAAO,KAAK,EAAE,eAAe,EAAE,aAAa,EAAE,YAAY,EAAE,MAAM,aAAa,CAAA;AAG/E,MAAM,MAAM,0BAA0B,GAAG;IACvC,MAAM,EAAE,MAAM,CAAA;IACd,OAAO,EAAE,aAAa,CAAA;IACtB,SAAS,EAAE,eAAe,CAAA;IAC1B,MAAM,EAAE,YAAY,CAAA;IACpB,MAAM,EAAE,YAAY,CAAA;IACpB,KAAK,EAAE,OAAO,CAAA;CACf,CAAA;AAOD,wBAAsB,mBAAmB,CAAC,EACxC,MAAM,EACN,OAAO,EACP,SAAS,EACT,MAAM,EACN,MAAM,EACN,KAAK,GACN,EAAE,0BAA0B,mBA8C5B;AAOD,wBAAsB,sBAAsB,CAAC,EAC3C,OAAO,EACP,SAAS,EACT,MAAM,EACN,MAAM,EACN,KAAK,GACN,EAAE,0BAA0B,mBAqD5B"}
+{"version":3,"file":"create-datasets.d.ts","sourceRoot":"","sources":["../../../src/pipeline/create-datasets.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,MAAM,CAAA;AAInC,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAA;AACtD,OAAO,KAAK,EAAE,UAAU,EAAE,aAAa,EAAE,YAAY,EAAE,MAAM,aAAa,CAAA;AAG1E,MAAM,MAAM,0BAA0B,GAAG;IACvC,MAAM,EAAE,MAAM,CAAA;IACd,OAAO,EAAE,aAAa,CAAA;IACtB,GAAG,EAAE,UAAU,CAAA;IACf,MAAM,EAAE,YAAY,CAAA;IACpB,MAAM,EAAE,YAAY,CAAA;IACpB,KAAK,EAAE,OAAO,CAAA;CACf,CAAA;AAOD,wBAAsB,mBAAmB,CAAC,EAAE,MAAM,EAAE,OAAO,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,EAAE,0BAA0B,mBA8CpH;AAOD,wBAAsB,sBAAsB,CAAC,EAAE,OAAO,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,EAAE,0BAA0B,mBA8C/G"}

package/dist/src/pipeline/create-datasets.js

@@ -1,11 +1,11 @@
 import * as p from '@clack/prompts';
 import * as SP from '@filoz/synapse-core/sp';
 import { getPDPProvider } from '@filoz/synapse-core/sp-registry';
-import { eq } from 'drizzle-orm';
+import { getDataSet, toStringMetadata } from "../api.js";
 import { findRepairDataset } from "../db/find-repair-dataset.js";
 import { repairUpdate } from "../db/repair-update.js";
 import { hashLink } from "../utils.js";
-export async function ensureRepairDataset({ source, localDb, indexerDb, client, repair, payer, }) {
+export async function ensureRepairDataset({ source, localDb, api, client, repair, payer }) {
     const log = p.taskLog({
         title: 'Ensuring repair dataset',
     });
@@ -16,7 +16,7 @@ export async function ensureRepairDataset({ source, localDb, indexerDb, client,
         throw new Error(`Target provider ${repair.targetProviderId} not found or inactive`);
     let datasetId = null;
     const existingDatasetId = await findRepairDataset({
-        indexerDb,
+        api,
         providerId: repair.targetProviderId,
         payer,
         source,
@@ -50,7 +50,7 @@ export async function ensureRepairDataset({ source, localDb, indexerDb, client,
     });
     return datasetId;
 }
-export async function ensureReplicateDataset({ localDb, indexerDb, client, repair, payer, }) {
+export async function ensureReplicateDataset({ localDb, api, client, repair, payer }) {
     const log = p.taskLog({
         title: 'Ensuring replication dataset',
     });
@@ -66,14 +66,7 @@ export async function ensureReplicateDataset({ localDb, indexerDb, client, repai
         log.success(`Data set #${repair.targetDataSetId} already exists at ${provider.pdp.serviceURL}`);
         return repair.targetDataSetId;
     }
-    const schema = indexerDb._.fullSchema;
-    const sourceDataSet = await indexerDb.query.dataSets.findFirst({
-        where: eq(schema.dataSets.dataSetId, repair.repairDataSetId),
-        columns: {
-            metadata: true,
-            withCdn: true,
-        },
-    });
+    const sourceDataSet = await getDataSet(api, repair.repairDataSetId);
     if (!sourceDataSet) {
         throw new Error(`Source dataset ${repair.repairDataSetId} not found`);
     }
@@ -81,8 +74,8 @@ export async function ensureReplicateDataset({ localDb, indexerDb, client, repai
         payee: provider.payee,
         serviceURL: provider.pdp.serviceURL,
         payer,
-        cdn: sourceDataSet.withCdn,
-        metadata: sourceDataSet.metadata ?? undefined,
+        cdn: sourceDataSet.withCdn ?? false,
+        metadata: sourceDataSet.metadata ? toStringMetadata(sourceDataSet.metadata) : undefined,
     });
     log.message(`Waiting for data to be created at ${provider.pdp.serviceURL} ${hashLink(txHash, client.chain)}...`);
     const waitForResult = await SP.waitForCreateDataSet({

package/dist/src/pipeline/create-datasets.js.map

@@ -1 +1 @@
-{"version":3,"file":"create-datasets.js","sourceRoot":"","sources":["../../../src/pipeline/create-datasets.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,CAAC,MAAM,gBAAgB,CAAA;AACnC,OAAO,KAAK,EAAE,MAAM,wBAAwB,CAAA;AAC5C,OAAO,EAAE,cAAc,EAAE,MAAM,iCAAiC,CAAA;AAChE,OAAO,EAAE,EAAE,EAAE,MAAM,aAAa,CAAA;AAEhC,OAAO,EAAE,iBAAiB,EAAE,MAAM,8BAA8B,CAAA;AAChE,OAAO,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAA;AAGrD,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAA;AAgBtC,MAAM,CAAC,KAAK,UAAU,mBAAmB,CAAC,EACxC,MAAM,EACN,OAAO,EACP,SAAS,EACT,MAAM,EACN,MAAM,EACN,KAAK,GACsB;IAC3B,MAAM,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC;QACpB,KAAK,EAAE,yBAAyB;KACjC,CAAC,CAAA;IACF,MAAM,QAAQ,GAAG,MAAM,cAAc,CAAC,MAAM,EAAE;QAC5C,UAAU,EAAE,MAAM,CAAC,gBAAgB;KACpC,CAAC,CAAA;IAEF,IAAI,CAAC,QAAQ;QAAE,MAAM,IAAI,KAAK,CAAC,mBAAmB,MAAM,CAAC,gBAAgB,wBAAwB,CAAC,CAAA;IAElG,IAAI,SAAS,GAAkB,IAAI,CAAA;IAEnC,MAAM,iBAAiB,GAAG,MAAM,iBAAiB,CAAC;QAChD,SAAS;QACT,UAAU,EAAE,MAAM,CAAC,gBAAgB;QACnC,KAAK;QACL,MAAM;KACP,CAAC,CAAA;IAEF,IAAI,iBAAiB,EAAE,CAAC;QACtB,SAAS,GAAG,iBAAiB,CAAA;QAC7B,GAAG,CAAC,OAAO,CAAC,aAAa,SAAS,sBAAsB,QAAQ,CAAC,GAAG,CAAC,UAAU,EAAE,CAAC,CAAA;IACpF,CAAC;SAAM,CAAC;QACN,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,GAAG,MAAM,EAAE,CAAC,aAAa,CAAC,MAAM,EAAE;YAC3D,KAAK,EAAE,QAAQ,CAAC,KAAK;YACrB,UAAU,EAAE,QAAQ,CAAC,GAAG,CAAC,UAAU;YACnC,KAAK;YACL,GAAG,EAAE,KAAK;YACV,QAAQ,EAAE;gBACR,MAAM;gBACN,gBAAgB,EAAE,EAAE;aACrB;SACF,CAAC,CAAA;QACF,GAAG,CAAC,OAAO,CAAC,qCAAqC,QAAQ,CAAC,GAAG,CAAC,UAAU,IAAI,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAA;QAChH,MAAM,aAAa,GAAG,MAAM,EAAE,CAAC,oBAAoB,CAAC;YAClD,SAAS;SACV,CAAC,CAAA;QACF,SAAS,GAAG,aAAa,CAAC,SAAS,CAAA;QACnC,GAAG,CAAC,OAAO,CAAC,aAAa,SAAS,eAAe,QAAQ,CAAC,GAAG,CAAC,UAAU,EAAE,CAAC,CAAA;IAC7E,CAAC;IACD,MAAM,YAAY,CAAC;QACjB,OAAO;QACP,QAAQ,EAAE,MAAM,CAAC,EAAE;QACnB,eAAe,EAAE,SAAS;KAC3B,CAAC,CAAA;IACF,OAAO,SAAS,CAAA;AAClB,CAAC;AAOD,MAAM,CAAC,KAAK,UAAU,sBAAsB,CAAC,EAC3C,OAAO,EACP,SAAS,EACT,MAAM,EACN,MAAM,EACN,KAAK,GACsB;IAC3B,MAAM,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC;QACpB,KAAK,EAAE,8BAA8B;KACtC,CAAC,CAAA;IAEF,IAAI,MAAM,CAAC,eAAe,IAAI,IAAI,EAAE,CAAC;QACnC,MAAM,IAAI,KAAK,CAAC,2BAA2B,CAAC,CAAA;IAC9C,CAAC;IAED,MAAM,QAAQ,GAAG,MAAM,cAAc,CAAC,MAAM,EAAE;QAC5C,UAAU,EAAE,MAAM,CAAC,gBAAgB;KACpC,CAAC,CAAA;IAEF,IAAI,CAAC,QAAQ;QAAE,MAAM,IAAI,KAAK,CAAC,mBAAmB,MAAM,CAAC,gBAAgB,wBAAwB,CAAC,CAAA;IAElG,IAAI,MAAM,CAAC,eAAe,IAAI,IAAI,EAAE,CAAC;QACnC,GAAG,CAAC,OAAO,CAAC,aAAa,MAAM,CAAC,eAAe,sBAAsB,QAAQ,CAAC,GAAG,CAAC,UAAU,EAAE,CAAC,CAAA;QAC/F,OAAO,MAAM,CAAC,eAAe,CAAA;IAC/B,CAAC;IAED,MAAM,MAAM,GAAG,SAAS,CAAC,CAAC,CAAC,UAAU,CAAA;IACrC,MAAM,aAAa,GAAG,MAAM,SAAS,CAAC,KAAK,CAAC,QAAQ,CAAC,SAAS,CAAC;QAC7D,KAAK,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC,eAAe,CAAC;QAC5D,OAAO,EAAE;YACP,QAAQ,EAAE,IAAI;YACd,OAAO,EAAE,IAAI;SACd;KACF,CAAC,CAAA;IAEF,IAAI,CAAC,aAAa,EAAE,CAAC;QACnB,MAAM,IAAI,KAAK,CAAC,kBAAkB,MAAM,CAAC,eAAe,YAAY,CAAC,CAAA;IACvE,CAAC;IAED,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,GAAG,MAAM,EAAE,CAAC,aAAa,CAAC,MAAM,EAAE;QAC3D,KAAK,EAAE,QAAQ,CAAC,KAAK;QACrB,UAAU,EAAE,QAAQ,CAAC,GAAG,CAAC,UAAU;QACnC,KAAK;QACL,GAAG,EAAE,aAAa,CAAC,OAAO;QAC1B,QAAQ,EAAE,aAAa,CAAC,QAAQ,IAAI,SAAS;KAC9C,CAAC,CAAA;IACF,GAAG,CAAC,OAAO,CAAC,qCAAqC,QAAQ,CAAC,GAAG,CAAC,UAAU,IAAI,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAA;IAChH,MAAM,aAAa,GAAG,MAAM,EAAE,CAAC,oBAAoB,CAAC;QAClD,SAAS;KACV,CAAC,CAAA;IACF,MAAM,SAAS,GAAG,aAAa,CAAC,SAAS,CAAA;IACzC,GAAG,CAAC,OAAO,CAAC,aAAa,SAAS,eAAe,QAAQ,CAAC,GAAG,CAAC,UAAU,EAAE,CAAC,CAAA;IAE3E,MAAM,YAAY,CAAC;QACjB,OAAO;QACP,QAAQ,EAAE,MAAM,CAAC,EAAE;QACnB,eAAe,EAAE,SAAS;KAC3B,CAAC,CAAA;IACF,OAAO,SAAS,CAAA;AAClB,CAAC"}
+{"version":3,"file":"create-datasets.js","sourceRoot":"","sources":["../../../src/pipeline/create-datasets.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,CAAC,MAAM,gBAAgB,CAAA;AACnC,OAAO,KAAK,EAAE,MAAM,wBAAwB,CAAA;AAC5C,OAAO,EAAE,cAAc,EAAE,MAAM,iCAAiC,CAAA;AAEhE,OAAO,EAAE,UAAU,EAAE,gBAAgB,EAAE,MAAM,WAAW,CAAA;AACxD,OAAO,EAAE,iBAAiB,EAAE,MAAM,8BAA8B,CAAA;AAChE,OAAO,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAA;AAGrD,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAA;AAgBtC,MAAM,CAAC,KAAK,UAAU,mBAAmB,CAAC,EAAE,MAAM,EAAE,OAAO,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAA8B;IACnH,MAAM,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC;QACpB,KAAK,EAAE,yBAAyB;KACjC,CAAC,CAAA;IACF,MAAM,QAAQ,GAAG,MAAM,cAAc,CAAC,MAAM,EAAE;QAC5C,UAAU,EAAE,MAAM,CAAC,gBAAgB;KACpC,CAAC,CAAA;IAEF,IAAI,CAAC,QAAQ;QAAE,MAAM,IAAI,KAAK,CAAC,mBAAmB,MAAM,CAAC,gBAAgB,wBAAwB,CAAC,CAAA;IAElG,IAAI,SAAS,GAAkB,IAAI,CAAA;IAEnC,MAAM,iBAAiB,GAAG,MAAM,iBAAiB,CAAC;QAChD,GAAG;QACH,UAAU,EAAE,MAAM,CAAC,gBAAgB;QACnC,KAAK;QACL,MAAM;KACP,CAAC,CAAA;IAEF,IAAI,iBAAiB,EAAE,CAAC;QACtB,SAAS,GAAG,iBAAiB,CAAA;QAC7B,GAAG,CAAC,OAAO,CAAC,aAAa,SAAS,sBAAsB,QAAQ,CAAC,GAAG,CAAC,UAAU,EAAE,CAAC,CAAA;IACpF,CAAC;SAAM,CAAC;QACN,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,GAAG,MAAM,EAAE,CAAC,aAAa,CAAC,MAAM,EAAE;YAC3D,KAAK,EAAE,QAAQ,CAAC,KAAK;YACrB,UAAU,EAAE,QAAQ,CAAC,GAAG,CAAC,UAAU;YACnC,KAAK;YACL,GAAG,EAAE,KAAK;YACV,QAAQ,EAAE;gBACR,MAAM;gBACN,gBAAgB,EAAE,EAAE;aACrB;SACF,CAAC,CAAA;QACF,GAAG,CAAC,OAAO,CAAC,qCAAqC,QAAQ,CAAC,GAAG,CAAC,UAAU,IAAI,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAA;QAChH,MAAM,aAAa,GAAG,MAAM,EAAE,CAAC,oBAAoB,CAAC;YAClD,SAAS;SACV,CAAC,CAAA;QACF,SAAS,GAAG,aAAa,CAAC,SAAS,CAAA;QACnC,GAAG,CAAC,OAAO,CAAC,aAAa,SAAS,eAAe,QAAQ,CAAC,GAAG,CAAC,UAAU,EAAE,CAAC,CAAA;IAC7E,CAAC;IACD,MAAM,YAAY,CAAC;QACjB,OAAO;QACP,QAAQ,EAAE,MAAM,CAAC,EAAE;QACnB,eAAe,EAAE,SAAS;KAC3B,CAAC,CAAA;IACF,OAAO,SAAS,CAAA;AAClB,CAAC;AAOD,MAAM,CAAC,KAAK,UAAU,sBAAsB,CAAC,EAAE,OAAO,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAA8B;IAC9G,MAAM,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC;QACpB,KAAK,EAAE,8BAA8B;KACtC,CAAC,CAAA;IAEF,IAAI,MAAM,CAAC,eAAe,IAAI,IAAI,EAAE,CAAC;QACnC,MAAM,IAAI,KAAK,CAAC,2BAA2B,CAAC,CAAA;IAC9C,CAAC;IAED,MAAM,QAAQ,GAAG,MAAM,cAAc,CAAC,MAAM,EAAE;QAC5C,UAAU,EAAE,MAAM,CAAC,gBAAgB;KACpC,CAAC,CAAA;IAEF,IAAI,CAAC,QAAQ;QAAE,MAAM,IAAI,KAAK,CAAC,mBAAmB,MAAM,CAAC,gBAAgB,wBAAwB,CAAC,CAAA;IAElG,IAAI,MAAM,CAAC,eAAe,IAAI,IAAI,EAAE,CAAC;QACnC,GAAG,CAAC,OAAO,CAAC,aAAa,MAAM,CAAC,eAAe,sBAAsB,QAAQ,CAAC,GAAG,CAAC,UAAU,EAAE,CAAC,CAAA;QAC/F,OAAO,MAAM,CAAC,eAAe,CAAA;IAC/B,CAAC;IAED,MAAM,aAAa,GAAG,MAAM,UAAU,CAAC,GAAG,EAAE,MAAM,CAAC,eAAe,CAAC,CAAA;IAEnE,IAAI,CAAC,aAAa,EAAE,CAAC;QACnB,MAAM,IAAI,KAAK,CAAC,kBAAkB,MAAM,CAAC,eAAe,YAAY,CAAC,CAAA;IACvE,CAAC;IAED,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,GAAG,MAAM,EAAE,CAAC,aAAa,CAAC,MAAM,EAAE;QAC3D,KAAK,EAAE,QAAQ,CAAC,KAAK;QACrB,UAAU,EAAE,QAAQ,CAAC,GAAG,CAAC,UAAU;QACnC,KAAK;QACL,GAAG,EAAE,aAAa,CAAC,OAAO,IAAI,KAAK;QACnC,QAAQ,EAAE,aAAa,CAAC,QAAQ,CAAC,CAAC,CAAC,gBAAgB,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,SAAS;KACxF,CAAC,CAAA;IACF,GAAG,CAAC,OAAO,CAAC,qCAAqC,QAAQ,CAAC,GAAG,CAAC,UAAU,IAAI,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAA;IAChH,MAAM,aAAa,GAAG,MAAM,EAAE,CAAC,oBAAoB,CAAC;QAClD,SAAS;KACV,CAAC,CAAA;IACF,MAAM,SAAS,GAAG,aAAa,CAAC,SAAS,CAAA;IACzC,GAAG,CAAC,OAAO,CAAC,aAAa,SAAS,eAAe,QAAQ,CAAC,GAAG,CAAC,UAAU,EAAE,CAAC,CAAA;IAE3E,MAAM,YAAY,CAAC;QACjB,OAAO;QACP,QAAQ,EAAE,MAAM,CAAC,EAAE;QACnB,eAAe,EAAE,SAAS;KAC3B,CAAC,CAAA;IACF,OAAO,SAAS,CAAA;AAClB,CAAC"}

package/dist/src/types.d.ts

@@ -1,23 +1,22 @@
+import type { ApiClient } from '@filecoin-project/api-client';
 import type { Chain } from '@filoz/synapse-core/chains';
 import type { Client } from '@libsql/client';
 import type { LibSQLDatabase } from 'drizzle-orm/libsql';
-import type { NodePgDatabase } from 'drizzle-orm/node-postgres';
 import type { z } from 'incur';
-import type { Pool } from 'pg';
 import type { Account, Address, Hex, Transport, Client as ViemClient } from 'viem';
-import type * as indexerSchema from './indexer-schema.ts';
+import type { Network } from './api.ts';
 import type * as localSchema from './local-schema.ts';
 import type { contextSchema } from './middleware.ts';
 export type LocalDatabase = LibSQLDatabase<typeof localSchema> & {
     $client: Client;
 };
-export type IndexerDatabase = NodePgDatabase<typeof indexerSchema> & {
-    $client: Pool;
+export type IndexerApi = {
+    client: ApiClient;
+    network: Network;
 };
 export interface Config {
     privateKey: Hex;
-    indexerMainnetUrl: string;
-    indexerCalibrationUrl: string;
+    apiUrl: string;
     chainId: number;
     dbPath: string;
 }

package/dist/src/types.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,4BAA4B,CAAA;AACvD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,gBAAgB,CAAA;AAC5C,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAA;AACxD,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,2BAA2B,CAAA;AAC/D,OAAO,KAAK,EAAE,CAAC,EAAE,MAAM,OAAO,CAAA;AAC9B,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,IAAI,CAAA;AAC9B,OAAO,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,GAAG,EAAE,SAAS,EAAE,MAAM,IAAI,UAAU,EAAE,MAAM,MAAM,CAAA;AAClF,OAAO,KAAK,KAAK,aAAa,MAAM,qBAAqB,CAAA;AACzD,OAAO,KAAK,KAAK,WAAW,MAAM,mBAAmB,CAAA;AACrD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAA;AACpD,MAAM,MAAM,aAAa,GAAG,cAAc,CAAC,OAAO,WAAW,CAAC,GAAG;IAC/D,OAAO,EAAE,MAAM,CAAA;CAChB,CAAA;AAED,MAAM,MAAM,eAAe,GAAG,cAAc,CAAC,OAAO,aAAa,CAAC,GAAG;IACnE,OAAO,EAAE,IAAI,CAAA;CACd,CAAA;AAED,MAAM,WAAW,MAAM;IACrB,UAAU,EAAE,GAAG,CAAA;IACf,iBAAiB,EAAE,MAAM,CAAA;IACzB,qBAAqB,EAAE,MAAM,CAAA;IAC7B,OAAO,EAAE,MAAM,CAAA;IACf,MAAM,EAAE,MAAM,CAAA;CACf;AAED,MAAM,MAAM,YAAY,GAAG,UAAU,CAAC,SAAS,EAAE,KAAK,EAAE,OAAO,CAAC,CAAA;AAEhE,MAAM,MAAM,OAAO,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,aAAa,CAAC,CAAA;AAKnD,MAAM,MAAM,cAAc,GAAG;IAC3B,UAAU,EAAE,MAAM,CAAA;IAClB,eAAe,EAAE,OAAO,CAAA;IACxB,IAAI,EAAE,MAAM,CAAA;IACZ,UAAU,EAAE,MAAM,CAAA;IAClB,QAAQ,EAAE,OAAO,CAAA;IACjB,QAAQ,EAAE,OAAO,CAAA;CAClB,CAAA"}
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,8BAA8B,CAAA;AAC7D,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,4BAA4B,CAAA;AACvD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,gBAAgB,CAAA;AAC5C,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAA;AACxD,OAAO,KAAK,EAAE,CAAC,EAAE,MAAM,OAAO,CAAA;AAC9B,OAAO,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,GAAG,EAAE,SAAS,EAAE,MAAM,IAAI,UAAU,EAAE,MAAM,MAAM,CAAA;AAClF,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,UAAU,CAAA;AACvC,OAAO,KAAK,KAAK,WAAW,MAAM,mBAAmB,CAAA;AACrD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAA;AACpD,MAAM,MAAM,aAAa,GAAG,cAAc,CAAC,OAAO,WAAW,CAAC,GAAG;IAC/D,OAAO,EAAE,MAAM,CAAA;CAChB,CAAA;AAKD,MAAM,MAAM,UAAU,GAAG;IACvB,MAAM,EAAE,SAAS,CAAA;IACjB,OAAO,EAAE,OAAO,CAAA;CACjB,CAAA;AAED,MAAM,WAAW,MAAM;IACrB,UAAU,EAAE,GAAG,CAAA;IACf,MAAM,EAAE,MAAM,CAAA;IACd,OAAO,EAAE,MAAM,CAAA;IACf,MAAM,EAAE,MAAM,CAAA;CACf;AAED,MAAM,MAAM,YAAY,GAAG,UAAU,CAAC,SAAS,EAAE,KAAK,EAAE,OAAO,CAAC,CAAA;AAEhE,MAAM,MAAM,OAAO,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,aAAa,CAAC,CAAA;AAKnD,MAAM,MAAM,cAAc,GAAG;IAC3B,UAAU,EAAE,MAAM,CAAA;IAClB,eAAe,EAAE,OAAO,CAAA;IACxB,IAAI,EAAE,MAAM,CAAA;IACZ,UAAU,EAAE,MAAM,CAAA;IAClB,QAAQ,EAAE,OAAO,CAAA;IACjB,QAAQ,EAAE,OAAO,CAAA;CAClB,CAAA"}