@filoz/repair-cli 0.4.0 → 1.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (115) hide show
  1. package/dist/package.json +4 -4
  2. package/dist/src/api.d.ts +35 -0
  3. package/dist/src/api.d.ts.map +1 -0
  4. package/dist/src/api.js +118 -0
  5. package/dist/src/api.js.map +1 -0
  6. package/dist/src/commands/datasets.d.ts +5 -7
  7. package/dist/src/commands/datasets.d.ts.map +1 -1
  8. package/dist/src/commands/datasets.js +33 -35
  9. package/dist/src/commands/datasets.js.map +1 -1
  10. package/dist/src/commands/providers.d.ts +5 -7
  11. package/dist/src/commands/providers.d.ts.map +1 -1
  12. package/dist/src/commands/providers.js +32 -33
  13. package/dist/src/commands/providers.js.map +1 -1
  14. package/dist/src/commands/repair.d.ts +5 -7
  15. package/dist/src/commands/repair.d.ts.map +1 -1
  16. package/dist/src/commands/repair.js +5 -1
  17. package/dist/src/commands/repair.js.map +1 -1
  18. package/dist/src/commands/replicate.d.ts +5 -7
  19. package/dist/src/commands/replicate.d.ts.map +1 -1
  20. package/dist/src/commands/replicate.js +5 -1
  21. package/dist/src/commands/replicate.js.map +1 -1
  22. package/dist/src/commands/second-copy.d.ts +5 -7
  23. package/dist/src/commands/second-copy.d.ts.map +1 -1
  24. package/dist/src/commands/second-copy.js +5 -1
  25. package/dist/src/commands/second-copy.js.map +1 -1
  26. package/dist/src/commands/session-key.d.ts +5 -7
  27. package/dist/src/commands/session-key.d.ts.map +1 -1
  28. package/dist/src/commands/setup.d.ts.map +1 -1
  29. package/dist/src/commands/setup.js +11 -32
  30. package/dist/src/commands/setup.js.map +1 -1
  31. package/dist/src/commands/wallet.d.ts +5 -7
  32. package/dist/src/commands/wallet.d.ts.map +1 -1
  33. package/dist/src/db/dedupe-cids.d.ts +6 -7
  34. package/dist/src/db/dedupe-cids.d.ts.map +1 -1
  35. package/dist/src/db/dedupe-cids.js +2 -14
  36. package/dist/src/db/dedupe-cids.js.map +1 -1
  37. package/dist/src/db/find-repair-dataset.d.ts +3 -3
  38. package/dist/src/db/find-repair-dataset.d.ts.map +1 -1
  39. package/dist/src/db/find-repair-dataset.js +16 -14
  40. package/dist/src/db/find-repair-dataset.js.map +1 -1
  41. package/dist/src/db/get-active-data-set-ids.d.ts +9 -0
  42. package/dist/src/db/get-active-data-set-ids.d.ts.map +1 -0
  43. package/dist/src/db/get-active-data-set-ids.js +9 -0
  44. package/dist/src/db/get-active-data-set-ids.js.map +1 -0
  45. package/dist/src/db/get-alternate-provider-urls.d.ts +6 -0
  46. package/dist/src/db/get-alternate-provider-urls.d.ts.map +1 -0
  47. package/dist/src/db/get-alternate-provider-urls.js +24 -0
  48. package/dist/src/db/get-alternate-provider-urls.js.map +1 -0
  49. package/dist/src/db/get-pieces.d.ts +8 -27
  50. package/dist/src/db/get-pieces.d.ts.map +1 -1
  51. package/dist/src/db/get-pieces.js +44 -97
  52. package/dist/src/db/get-pieces.js.map +1 -1
  53. package/dist/src/db/get-repair-provider.d.ts +2 -2
  54. package/dist/src/db/get-repair-provider.d.ts.map +1 -1
  55. package/dist/src/db/get-repair-provider.js +8 -20
  56. package/dist/src/db/get-repair-provider.js.map +1 -1
  57. package/dist/src/db/repair-create.js +3 -3
  58. package/dist/src/db/repair-create.js.map +1 -1
  59. package/dist/src/db/replicate-create.d.ts.map +1 -1
  60. package/dist/src/db/replicate-create.js +13 -21
  61. package/dist/src/db/replicate-create.js.map +1 -1
  62. package/dist/src/db/second-copy-create.js +2 -2
  63. package/dist/src/db/second-copy-create.js.map +1 -1
  64. package/dist/src/local-schema.d.ts +1 -1
  65. package/dist/src/middleware.d.ts +12 -16
  66. package/dist/src/middleware.d.ts.map +1 -1
  67. package/dist/src/middleware.js +6 -10
  68. package/dist/src/middleware.js.map +1 -1
  69. package/dist/src/pipeline/add-pieces.d.ts +4 -4
  70. package/dist/src/pipeline/add-pieces.d.ts.map +1 -1
  71. package/dist/src/pipeline/add-pieces.js +32 -19
  72. package/dist/src/pipeline/add-pieces.js.map +1 -1
  73. package/dist/src/pipeline/create-datasets.d.ts +4 -4
  74. package/dist/src/pipeline/create-datasets.d.ts.map +1 -1
  75. package/dist/src/pipeline/create-datasets.js +7 -14
  76. package/dist/src/pipeline/create-datasets.js.map +1 -1
  77. package/dist/src/types.d.ts +8 -9
  78. package/dist/src/types.d.ts.map +1 -1
  79. package/dist/src/utils.d.ts +45 -37
  80. package/dist/src/utils.d.ts.map +1 -1
  81. package/dist/src/utils.js +47 -2
  82. package/dist/src/utils.js.map +1 -1
  83. package/package.json +4 -4
  84. package/readme.md +22 -19
  85. package/src/api.ts +220 -0
  86. package/src/commands/datasets.ts +33 -38
  87. package/src/commands/providers.ts +34 -42
  88. package/src/commands/repair.ts +5 -1
  89. package/src/commands/replicate.ts +5 -1
  90. package/src/commands/second-copy.ts +5 -1
  91. package/src/commands/setup.ts +12 -35
  92. package/src/db/dedupe-cids.ts +7 -24
  93. package/src/db/find-repair-dataset.ts +17 -28
  94. package/src/db/get-active-data-set-ids.ts +29 -0
  95. package/src/db/get-alternate-provider-urls.ts +43 -0
  96. package/src/db/get-pieces.ts +82 -214
  97. package/src/db/get-repair-provider.ts +10 -30
  98. package/src/db/repair-create.ts +3 -3
  99. package/src/db/replicate-create.ts +13 -21
  100. package/src/db/second-copy-create.ts +2 -2
  101. package/src/middleware.ts +10 -14
  102. package/src/pipeline/add-pieces.ts +52 -24
  103. package/src/pipeline/create-datasets.ts +9 -29
  104. package/src/types.ts +11 -9
  105. package/src/utils.ts +71 -7
  106. package/dist/src/db/find-providers-by-cid.d.ts +0 -9
  107. package/dist/src/db/find-providers-by-cid.d.ts.map +0 -1
  108. package/dist/src/db/find-providers-by-cid.js +0 -43
  109. package/dist/src/db/find-providers-by-cid.js.map +0 -1
  110. package/dist/src/indexer-schema.d.ts +0 -549
  111. package/dist/src/indexer-schema.d.ts.map +0 -1
  112. package/dist/src/indexer-schema.js +0 -56
  113. package/dist/src/indexer-schema.js.map +0 -1
  114. package/src/db/find-providers-by-cid.ts +0 -70
  115. package/src/indexer-schema.ts +0 -77
@@ -1,9 +1,8 @@
1
- import { drizzle as drizzlePostgres } from 'drizzle-orm/node-postgres';
2
1
  import { middleware, z } from 'incur';
3
- import * as indexerSchema from "./indexer-schema.js";
2
+ import { createIndexerApi } from "./api.js";
4
3
  import { config, createLocalDatabase, getClient } from "./utils.js";
5
4
  export const contextSchema = z.object({
6
- indexerDb: z.custom(),
5
+ api: z.custom(),
7
6
  localDb: z.custom(),
8
7
  config: z.custom(),
9
8
  client: z.custom(),
@@ -12,8 +11,8 @@ export const contextSchema = z.object({
12
11
  isInteractive: z.boolean(),
13
12
  });
14
13
  export const contextMiddleware = middleware(async (c, next) => {
15
- const { dbPath, chainId, indexerMainnetUrl, indexerCalibrationUrl, source } = config.store;
16
- if (!dbPath || !chainId || !indexerMainnetUrl || !indexerCalibrationUrl || !source) {
14
+ const { dbPath, chainId, apiUrl, source } = config.store;
15
+ if (!dbPath || !chainId || !source) {
17
16
  return c.error({
18
17
  code: 'CONFIG_NOT_SET',
19
18
  message: 'Config not set. Please run `repair setup` first.',
@@ -21,12 +20,10 @@ export const contextMiddleware = middleware(async (c, next) => {
21
20
  });
22
21
  }
23
22
  const localDb = await createLocalDatabase(dbPath);
24
- const indexerDb = drizzlePostgres(chainId === 314 ? indexerMainnetUrl : indexerCalibrationUrl, {
25
- schema: indexerSchema,
26
- });
23
+ const api = createIndexerApi({ baseUrl: apiUrl, chainId });
27
24
  const { client, chain } = getClient(chainId);
28
25
  c.set('localDb', localDb);
29
- c.set('indexerDb', indexerDb);
26
+ c.set('api', api);
30
27
  c.set('config', config);
31
28
  c.set('client', client);
32
29
  c.set('chain', chain);
@@ -34,6 +31,5 @@ export const contextMiddleware = middleware(async (c, next) => {
34
31
  c.set('isInteractive', !c.agent && !c.formatExplicit);
35
32
  await next();
36
33
  localDb.$client.close();
37
- await indexerDb.$client.end();
38
34
  });
39
35
  //# sourceMappingURL=middleware.js.map
@@ -1 +1 @@
1
- {"version":3,"file":"middleware.js","sourceRoot":"","sources":["../../src/middleware.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,OAAO,IAAI,eAAe,EAAE,MAAM,2BAA2B,CAAA;AACtE,OAAO,EAAE,UAAU,EAAE,CAAC,EAAE,MAAM,OAAO,CAAA;AAErC,OAAO,KAAK,aAAa,MAAM,qBAAqB,CAAA;AAEpD,OAAO,EAAE,MAAM,EAAE,mBAAmB,EAAE,SAAS,EAAE,MAAM,YAAY,CAAA;AAEnE,MAAM,CAAC,MAAM,aAAa,GAAG,CAAC,CAAC,MAAM,CAAC;IACpC,SAAS,EAAE,CAAC,CAAC,MAAM,EAAmB;IACtC,OAAO,EAAE,CAAC,CAAC,MAAM,EAAiB;IAClC,MAAM,EAAE,CAAC,CAAC,MAAM,EAAiB;IACjC,MAAM,EAAE,CAAC,CAAC,MAAM,EAAqC;IACrD,KAAK,EAAE,CAAC,CAAC,MAAM,EAAS;IACxB,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE;IAClB,aAAa,EAAE,CAAC,CAAC,OAAO,EAAE;CAC3B,CAAC,CAAA;AAEF,MAAM,CAAC,MAAM,iBAAiB,GAAG,UAAU,CAAuB,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,EAAE;IAClF,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,iBAAiB,EAAE,qBAAqB,EAAE,MAAM,EAAE,GAAG,MAAM,CAAC,KAAK,CAAA;IAE1F,IAAI,CAAC,MAAM,IAAI,CAAC,OAAO,IAAI,CAAC,iBAAiB,IAAI,CAAC,qBAAqB,IAAI,CAAC,MAAM,EAAE,CAAC;QACnF,OAAO,CAAC,CAAC,KAAK,CAAC;YACb,IAAI,EAAE,gBAAgB;YACtB,OAAO,EAAE,kDAAkD;YAC3D,SAAS,EAAE,KAAK;SACjB,CAAC,CAAA;IACJ,CAAC;IAED,MAAM,OAAO,GAAG,MAAM,mBAAmB,CAAC,MAAM,CAAC,CAAA;IACjD,MAAM,SAAS,GAAG,eAAe,CAAC,OAAO,KAAK,GAAG,CAAC,CAAC,CAAC,iBAAiB,CAAC,CAAC,CAAC,qBAAqB,EAAE;QAC7F,MAAM,EAAE,aAAa;KACtB,CAAC,CAAA;IAEF,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG,SAAS,CAAC,OAAO,CAAC,CAAA;IAC5C,CAAC,CAAC,GAAG,CAAC,SAAS,EAAE,OAAO,CAAC,CAAA;IACzB,CAAC,CAAC,GAAG,CAAC,WAAW,EAAE,SAAS,CAAC,CAAA;IAC7B,CAAC,CAAC,GAAG,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAA;IACvB,CAAC,CAAC,GAAG,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAA;IACvB,CAAC,CAAC,GAAG,CAAC,OAAO,EAAE,KAAK,CAAC,CAAA;IACrB,CAAC,CAAC,GAAG,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAA;IACvB,CAAC,CAAC,GAAG,CAAC,eAAe,EAAE,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC,cAAc,CAAC,CAAA;IACrD,MAAM,IAAI,EAAE,CAAA;IAEZ,OAAO,CAAC,OAAO,CAAC,KAAK,EAAE,CAAA;IACvB,MAAM,SAAS,CAAC,OAAO,CAAC,GAAG,EAAE,CAAA;AAC/B,CAAC,CAAC,CAAA"}
1
+ {"version":3,"file":"middleware.js","sourceRoot":"","sources":["../../src/middleware.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,UAAU,EAAE,CAAC,EAAE,MAAM,OAAO,CAAA;AAErC,OAAO,EAAE,gBAAgB,EAAE,MAAM,UAAU,CAAA;AAE3C,OAAO,EAAE,MAAM,EAAE,mBAAmB,EAAE,SAAS,EAAE,MAAM,YAAY,CAAA;AAEnE,MAAM,CAAC,MAAM,aAAa,GAAG,CAAC,CAAC,MAAM,CAAC;IACpC,GAAG,EAAE,CAAC,CAAC,MAAM,EAAc;IAC3B,OAAO,EAAE,CAAC,CAAC,MAAM,EAAiB;IAClC,MAAM,EAAE,CAAC,CAAC,MAAM,EAAiB;IACjC,MAAM,EAAE,CAAC,CAAC,MAAM,EAA6C;IAC7D,KAAK,EAAE,CAAC,CAAC,MAAM,EAAiB;IAChC,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE;IAClB,aAAa,EAAE,CAAC,CAAC,OAAO,EAAE;CAC3B,CAAC,CAAA;AAEF,MAAM,CAAC,MAAM,iBAAiB,GAAG,UAAU,CAAuB,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,EAAE;IAClF,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,CAAC,KAAK,CAAA;IAExD,IAAI,CAAC,MAAM,IAAI,CAAC,OAAO,IAAI,CAAC,MAAM,EAAE,CAAC;QACnC,OAAO,CAAC,CAAC,KAAK,CAAC;YACb,IAAI,EAAE,gBAAgB;YACtB,OAAO,EAAE,kDAAkD;YAC3D,SAAS,EAAE,KAAK;SACjB,CAAC,CAAA;IACJ,CAAC;IAED,MAAM,OAAO,GAAG,MAAM,mBAAmB,CAAC,MAAM,CAAC,CAAA;IACjD,MAAM,GAAG,GAAG,gBAAgB,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC,CAAA;IAE1D,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG,SAAS,CAAC,OAAO,CAAC,CAAA;IAC5C,CAAC,CAAC,GAAG,CAAC,SAAS,EAAE,OAAO,CAAC,CAAA;IACzB,CAAC,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,CAAC,CAAA;IACjB,CAAC,CAAC,GAAG,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAA;IACvB,CAAC,CAAC,GAAG,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAA;IACvB,CAAC,CAAC,GAAG,CAAC,OAAO,EAAE,KAAK,CAAC,CAAA;IACrB,CAAC,CAAC,GAAG,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAA;IACvB,CAAC,CAAC,GAAG,CAAC,eAAe,EAAE,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC,cAAc,CAAC,CAAA;IACrD,MAAM,IAAI,EAAE,CAAA;IAEZ,OAAO,CAAC,OAAO,CAAC,KAAK,EAAE,CAAA;AACzB,CAAC,CAAC,CAAA"}
@@ -1,12 +1,12 @@
1
1
  import type { RepairSelect } from '../local-schema.ts';
2
- import type { IndexerDatabase, LocalDatabase, WalletClient } from '../types.ts';
2
+ import type { IndexerApi, LocalDatabase, WalletClient } from '../types.ts';
3
3
  export type RunPullPiecesPhaseOptions = {
4
4
  localDb: LocalDatabase;
5
- indexerDb: IndexerDatabase;
5
+ api: IndexerApi;
6
6
  repair: RepairSelect;
7
7
  concurrency: number;
8
- batchSize: number;
8
+ batchSize?: number;
9
9
  client: WalletClient;
10
10
  };
11
- export declare function runAddPieces({ localDb, indexerDb, repair, concurrency, batchSize, client, }: RunPullPiecesPhaseOptions): Promise<void>;
11
+ export declare function runAddPieces({ localDb, api, repair, concurrency, batchSize, client, }: RunPullPiecesPhaseOptions): Promise<void>;
12
12
  //# sourceMappingURL=add-pieces.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"add-pieces.d.ts","sourceRoot":"","sources":["../../../src/pipeline/add-pieces.ts"],"names":[],"mappings":"AASA,OAAO,KAAK,EAAmB,YAAY,EAAE,MAAM,oBAAoB,CAAA;AACvE,OAAO,KAAK,EAAE,eAAe,EAAE,aAAa,EAAE,YAAY,EAAE,MAAM,aAAa,CAAA;AAG/E,MAAM,MAAM,yBAAyB,GAAG;IACtC,OAAO,EAAE,aAAa,CAAA;IACtB,SAAS,EAAE,eAAe,CAAA;IAC1B,MAAM,EAAE,YAAY,CAAA;IACpB,WAAW,EAAE,MAAM,CAAA;IACnB,SAAS,EAAE,MAAM,CAAA;IACjB,MAAM,EAAE,YAAY,CAAA;CACrB,CAAA;AA0ID,wBAAsB,YAAY,CAAC,EACjC,OAAO,EACP,SAAS,EACT,MAAM,EACN,WAAW,EACX,SAAS,EACT,MAAM,GACP,EAAE,yBAAyB,GAAG,OAAO,CAAC,IAAI,CAAC,CAiE3C"}
1
+ {"version":3,"file":"add-pieces.d.ts","sourceRoot":"","sources":["../../../src/pipeline/add-pieces.ts"],"names":[],"mappings":"AASA,OAAO,KAAK,EAAmB,YAAY,EAAE,MAAM,oBAAoB,CAAA;AACvE,OAAO,KAAK,EAAE,UAAU,EAAE,aAAa,EAAE,YAAY,EAAE,MAAM,aAAa,CAAA;AAa1E,MAAM,MAAM,yBAAyB,GAAG;IACtC,OAAO,EAAE,aAAa,CAAA;IACtB,GAAG,EAAE,UAAU,CAAA;IACf,MAAM,EAAE,YAAY,CAAA;IACpB,WAAW,EAAE,MAAM,CAAA;IAEnB,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,MAAM,EAAE,YAAY,CAAA;CACrB,CAAA;AAwID,wBAAsB,YAAY,CAAC,EACjC,OAAO,EACP,GAAG,EACH,MAAM,EACN,WAAW,EACX,SAAS,EACT,MAAM,GACP,EAAE,yBAAyB,GAAG,OAAO,CAAC,IAAI,CAAC,CAoF3C"}
@@ -1,24 +1,24 @@
1
1
  import { taskLog } from '@clack/prompts';
2
- import * as Piece from '@filoz/synapse-core/piece';
3
2
  import * as SP from '@filoz/synapse-core/sp';
4
3
  import { and, asc, eq, gt, inArray } from 'drizzle-orm';
5
4
  import PQueue from 'p-queue';
5
+ import { listDataSetPieces } from "../api.js";
6
6
  import { dedupeCids } from "../db/dedupe-cids.js";
7
7
  import { getTargetDataset } from "../db/get-target-dataset.js";
8
8
  import { repairUpdate } from "../db/repair-update.js";
9
9
  import { upsertOperations } from "../db/upsert-operations.js";
10
- import { completeConfirmedOperations, excludeOperationsByCid, hashLink, operationsToPullPieces } from "../utils.js";
11
- function createAddPiecesWorker({ localDb, indexerDb, repair, client, state, log }) {
10
+ import { completeConfirmedOperations, excludeOperationsByCid, fitAddPiecesBatch, hashLink, operationsToAddPieces, operationsToPullPieces, } from "../utils.js";
11
+ const MAX_BATCH_CANDIDATES = 500;
12
+ function createAddPiecesWorker({ localDb, targetCids, repair, client, state, log }) {
12
13
  return async (options) => {
13
14
  let completedOps = 0;
14
15
  let failedOps = 0;
15
16
  let operations = options.operations;
16
- const isRepair = repair.repairDataSetId == null;
17
- const group = log.group(`Batch ${options.batchNumber}/${state.totalBatches}`);
17
+ const group = log.group(`Batch ${options.batchNumber} (${options.operations.length} pieces)`);
18
18
  try {
19
19
  const dataset = await getTargetDataset({ localDb, repairId: repair.id, client });
20
- if (isRepair) {
21
- operations = await dedupeCids({ indexerDb, localDb, dataSetId: dataset.dataSetId, operations });
20
+ if (targetCids) {
21
+ operations = await dedupeCids({ localDb, existingCids: targetCids, operations });
22
22
  }
23
23
  const operationsBeforeConfirmationCheck = operations.length;
24
24
  operations = await completeConfirmedOperations({ localDb, client, operations });
@@ -34,7 +34,6 @@ function createAddPiecesWorker({ localDb, indexerDb, repair, client, state, log
34
34
  dataSetId: dataset.dataSetId,
35
35
  clientDataSetId: dataset.clientDataSetId,
36
36
  pieces: operationsToPullPieces(operations),
37
- timeout: 1000 * 60 * 30,
38
37
  onStatus: (status) => {
39
38
  const completed = status.pieces.filter((piece) => piece.status === 'complete').length;
40
39
  const failed = status.pieces.filter((piece) => piece.status === 'failed').length;
@@ -65,10 +64,7 @@ function createAddPiecesWorker({ localDb, indexerDb, repair, client, state, log
65
64
  serviceURL: repair.targetProviderUrl,
66
65
  dataSetId: dataset.dataSetId,
67
66
  clientDataSetId: dataset.clientDataSetId,
68
- pieces: operations.map((operation) => ({
69
- pieceCid: Piece.from(operation.cid),
70
- metadata: isRepair ? undefined : operation.metadata,
71
- })),
67
+ pieces: operationsToAddPieces(operations, !targetCids),
72
68
  });
73
69
  await upsertOperations({
74
70
  localDb,
@@ -79,15 +75,18 @@ function createAddPiecesWorker({ localDb, indexerDb, repair, client, state, log
79
75
  });
80
76
  group.message(`Waiting for add pieces ${hashLink(addPiecesResult.txHash, client.chain)}...`);
81
77
  const waitForAddPieces = await SP.waitForAddPieces(addPiecesResult);
78
+ const confirmedTxHash = waitForAddPieces.confirmedTxHash ?? waitForAddPieces.txHash;
82
79
  state.completedOperations += operations.length;
83
80
  completedOps += operations.length;
81
+ for (const operation of operations)
82
+ targetCids?.add(operation.cid);
84
83
  await upsertOperations({
85
84
  localDb,
86
85
  operations: operations.map((operation) => ({
87
86
  ...operation,
88
87
  status: 'completed',
89
88
  error: null,
90
- txHash: waitForAddPieces.txHash,
89
+ txHash: confirmedTxHash,
91
90
  })),
92
91
  });
93
92
  }
@@ -108,13 +107,12 @@ function createAddPiecesWorker({ localDb, indexerDb, repair, client, state, log
108
107
  }
109
108
  };
110
109
  }
111
- export async function runAddPieces({ localDb, indexerDb, repair, concurrency, batchSize, client, }) {
110
+ export async function runAddPieces({ localDb, api, repair, concurrency, batchSize, client, }) {
112
111
  const localSchema = localDb._.fullSchema;
113
112
  let cursor = 0;
114
113
  const totalOperations = await localDb.$count(localSchema.operations, and(eq(localSchema.operations.repairId, repair.id), eq(localSchema.operations.type, 'add_piece'), inArray(localSchema.operations.status, ['pending', 'failed'])));
115
114
  let batchNumber = 0;
116
115
  const state = {
117
- totalBatches: Math.ceil(totalOperations / batchSize),
118
116
  totalOperations,
119
117
  completedOperations: 0,
120
118
  failedOperations: 0,
@@ -123,19 +121,34 @@ export async function runAddPieces({ localDb, indexerDb, repair, concurrency, ba
123
121
  title: 'Adding pieces',
124
122
  limit: 1,
125
123
  });
124
+ const dataset = totalOperations > 0 ? await getTargetDataset({ localDb, repairId: repair.id, client }) : undefined;
125
+ let targetCids;
126
+ if (repair.repairDataSetId == null && dataset) {
127
+ const pieces = await listDataSetPieces(api, dataset.dataSetId, { removed: false });
128
+ targetCids = new Set(pieces.flatMap((piece) => (piece.cid == null ? [] : [piece.cid])));
129
+ }
126
130
  async function getNextBatch() {
127
- const operations = await localDb.query.operations.findMany({
131
+ if (!dataset) {
132
+ return null;
133
+ }
134
+ const candidates = await localDb.query.operations.findMany({
128
135
  where: and(eq(localSchema.operations.repairId, repair.id), eq(localSchema.operations.type, 'add_piece'), inArray(localSchema.operations.status, ['pending', 'failed']), gt(localSchema.operations.id, cursor)),
129
136
  orderBy: [asc(localSchema.operations.id)],
130
- limit: batchSize,
137
+ limit: batchSize ?? MAX_BATCH_CANDIDATES,
131
138
  });
132
- if (operations.length === 0) {
139
+ if (candidates.length === 0) {
133
140
  return null;
134
141
  }
142
+ const operations = fitAddPiecesBatch({
143
+ operations: candidates,
144
+ dataSetId: dataset.dataSetId,
145
+ chain: client.chain,
146
+ withMetadata: !targetCids,
147
+ });
135
148
  cursor = operations.at(-1)?.id ?? cursor;
136
149
  return operations;
137
150
  }
138
- const addPiecesJob = createAddPiecesWorker({ localDb, indexerDb, repair, client, state, log });
151
+ const addPiecesJob = createAddPiecesWorker({ localDb, targetCids, repair, client, state, log });
139
152
  const queue = new PQueue({ concurrency });
140
153
  while (true) {
141
154
  await queue.onSizeLessThan(concurrency);
@@ -1 +1 @@
1
- {"version":3,"file":"add-pieces.js","sourceRoot":"","sources":["../../../src/pipeline/add-pieces.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,gBAAgB,CAAA;AACxC,OAAO,KAAK,KAAK,MAAM,2BAA2B,CAAA;AAClD,OAAO,KAAK,EAAE,MAAM,wBAAwB,CAAA;AAC5C,OAAO,EAAE,GAAG,EAAE,GAAG,EAAE,EAAE,EAAE,EAAE,EAAE,OAAO,EAAE,MAAM,aAAa,CAAA;AACvD,OAAO,MAAM,MAAM,SAAS,CAAA;AAC5B,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAA;AACjD,OAAO,EAAE,gBAAgB,EAAE,MAAM,6BAA6B,CAAA;AAC9D,OAAO,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAA;AACrD,OAAO,EAAE,gBAAgB,EAAE,MAAM,4BAA4B,CAAA;AAG7D,OAAO,EAAE,2BAA2B,EAAE,sBAAsB,EAAE,QAAQ,EAAE,sBAAsB,EAAE,MAAM,aAAa,CAAA;AAiCnH,SAAS,qBAAqB,CAAC,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG,EAAgC;IAC7G,OAAO,KAAK,EAAE,OAA4B,EAAE,EAAE;QAC5C,IAAI,YAAY,GAAG,CAAC,CAAA;QACpB,IAAI,SAAS,GAAG,CAAC,CAAA;QACjB,IAAI,UAAU,GAAsB,OAAO,CAAC,UAAU,CAAA;QACtD,MAAM,QAAQ,GAAG,MAAM,CAAC,eAAe,IAAI,IAAI,CAAA;QAE/C,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC,SAAS,OAAO,CAAC,WAAW,IAAI,KAAK,CAAC,YAAY,EAAE,CAAC,CAAA;QAE7E,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,MAAM,gBAAgB,CAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,CAAC,EAAE,EAAE,MAAM,EAAE,CAAC,CAAA;YAGhF,IAAI,QAAQ,EAAE,CAAC;gBACb,UAAU,GAAG,MAAM,UAAU,CAAC,EAAE,SAAS,EAAE,OAAO,EAAE,SAAS,EAAE,OAAO,CAAC,SAAS,EAAE,UAAU,EAAE,CAAC,CAAA;YACjG,CAAC;YAED,MAAM,iCAAiC,GAAG,UAAU,CAAC,MAAM,CAAA;YAC3D,UAAU,GAAG,MAAM,2BAA2B,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC,CAAA;YAC/E,MAAM,mBAAmB,GAAG,iCAAiC,GAAG,UAAU,CAAC,MAAM,CAAA;YACjF,IAAI,mBAAmB,GAAG,CAAC,EAAE,CAAC;gBAC5B,KAAK,CAAC,mBAAmB,IAAI,mBAAmB,CAAA;gBAChD,YAAY,IAAI,mBAAmB,CAAA;YACrC,CAAC;YAED,KAAK,CAAC,OAAO,CAAC,WAAW,UAAU,CAAC,MAAM,YAAY,CAAC,CAAA;YAEvD,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC1B,MAAM,UAAU,GAAG,MAAM,EAAE,CAAC,iBAAiB,CAAC,MAAM,EAAE;oBACpD,UAAU,EAAE,MAAM,CAAC,iBAAiB;oBACpC,SAAS,EAAE,OAAO,CAAC,SAAS;oBAC5B,eAAe,EAAE,OAAO,CAAC,eAAe;oBACxC,MAAM,EAAE,sBAAsB,CAAC,UAAU,CAAC;oBAC1C,OAAO,EAAE,IAAI,GAAG,EAAE,GAAG,EAAE;oBACvB,QAAQ,EAAE,CAAC,MAAM,EAAE,EAAE;wBACnB,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,KAAK,UAAU,CAAC,CAAC,MAAM,CAAA;wBACrF,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,MAAM,CAAA;wBAChF,KAAK,CAAC,OAAO,CAAC,QAAQ,SAAS,eAAe,MAAM,SAAS,CAAC,CAAA;oBAChE,CAAC;iBACF,CAAC,CAAA;gBAEF,KAAK,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,UAAU,CAAC,MAAM,EAAE,CAAC;oBACrD,MAAM,GAAG,GAAG,QAAQ,CAAC,QAAQ,EAAE,CAAA;oBAC/B,IAAI,MAAM,KAAK,UAAU,EAAE,CAAC;wBAC1B,KAAK,CAAC,gBAAgB,EAAE,CAAA;wBACxB,SAAS,EAAE,CAAA;wBACX,MAAM,EAAE,iBAAiB,EAAE,iBAAiB,EAAE,GAAG,sBAAsB,CAAC,UAAU,EAAE,GAAG,CAAC,CAAA;wBACxF,UAAU,GAAG,iBAAiB,CAAA;wBAC9B,MAAM,gBAAgB,CAAC;4BACrB,OAAO;4BACP,UAAU,EAAE,iBAAiB,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;gCAChD,GAAG,SAAS;gCACZ,MAAM,EAAE,QAAQ;gCAChB,KAAK,EAAE,2BAA2B,MAAM,EAAE;6BAC3C,CAAC,CAAC;yBACJ,CAAC,CAAA;oBACJ,CAAC;gBACH,CAAC;YACH,CAAC;YAGD,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC1B,KAAK,CAAC,OAAO,CAAC,UAAU,UAAU,CAAC,MAAM,YAAY,CAAC,CAAA;gBACtD,MAAM,eAAe,GAAG,MAAM,EAAE,CAAC,SAAS,CAAC,MAAM,EAAE;oBACjD,UAAU,EAAE,MAAM,CAAC,iBAAiB;oBACpC,SAAS,EAAE,OAAO,CAAC,SAAS;oBAC5B,eAAe,EAAE,OAAO,CAAC,eAAe;oBACxC,MAAM,EAAE,UAAU,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;wBACrC,QAAQ,EAAE,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC;wBACnC,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,QAAQ;qBACpD,CAAC,CAAC;iBACJ,CAAC,CAAA;gBACF,MAAM,gBAAgB,CAAC;oBACrB,OAAO;oBACP,UAAU,EAAE,UAAU,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;wBACzC,GAAG,SAAS;wBACZ,MAAM,EAAE,eAAe,CAAC,MAAM;qBAC/B,CAAC,CAAC;iBACJ,CAAC,CAAA;gBAEF,KAAK,CAAC,OAAO,CAAC,0BAA0B,QAAQ,CAAC,eAAe,CAAC,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAA;gBAC5F,MAAM,gBAAgB,GAAG,MAAM,EAAE,CAAC,gBAAgB,CAAC,eAAe,CAAC,CAAA;gBACnE,KAAK,CAAC,mBAAmB,IAAI,UAAU,CAAC,MAAM,CAAA;gBAC9C,YAAY,IAAI,UAAU,CAAC,MAAM,CAAA;gBACjC,MAAM,gBAAgB,CAAC;oBACrB,OAAO;oBACP,UAAU,EAAE,UAAU,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;wBACzC,GAAG,SAAS;wBACZ,MAAM,EAAE,WAAW;wBACnB,KAAK,EAAE,IAAI;wBACX,MAAM,EAAE,gBAAgB,CAAC,MAAM;qBAChC,CAAC,CAAC;iBACJ,CAAC,CAAA;YACJ,CAAC;YACD,KAAK,CAAC,OAAO,CAAC,SAAS,YAAY,WAAW,SAAS,SAAS,CAAC,CAAA;QACnE,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,KAAK,CAAC,gBAAgB,IAAI,UAAU,CAAC,MAAM,CAAA;YAC3C,MAAM,OAAO,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,CAAA;YACxE,KAAK,CAAC,OAAO,CAAC,GAAG,OAAO,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,EAAE,CAAC,CAAA;YAC/C,MAAM,gBAAgB,CAAC;gBACrB,OAAO;gBACP,UAAU,EAAE,UAAU,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;oBACzC,GAAG,SAAS;oBACZ,MAAM,EAAE,QAAQ;oBAChB,KAAK,EAAE,OAAO;iBACf,CAAC,CAAC;aACJ,CAAC,CAAA;QACJ,CAAC;IACH,CAAC,CAAA;AACH,CAAC;AAKD,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,EACjC,OAAO,EACP,SAAS,EACT,MAAM,EACN,WAAW,EACX,SAAS,EACT,MAAM,GACoB;IAC1B,MAAM,WAAW,GAAG,OAAO,CAAC,CAAC,CAAC,UAAU,CAAA;IACxC,IAAI,MAAM,GAAG,CAAC,CAAA;IAEd,MAAM,eAAe,GAAG,MAAM,OAAO,CAAC,MAAM,CAC1C,WAAW,CAAC,UAAU,EACtB,GAAG,CACD,EAAE,CAAC,WAAW,CAAC,UAAU,CAAC,QAAQ,EAAE,MAAM,CAAC,EAAE,CAAC,EAC9C,EAAE,CAAC,WAAW,CAAC,UAAU,CAAC,IAAI,EAAE,WAAW,CAAC,EAC5C,OAAO,CAAC,WAAW,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC,CAC9D,CACF,CAAA;IACD,IAAI,WAAW,GAAG,CAAC,CAAA;IACnB,MAAM,KAAK,GAAG;QACZ,YAAY,EAAE,IAAI,CAAC,IAAI,CAAC,eAAe,GAAG,SAAS,CAAC;QACpD,eAAe;QACf,mBAAmB,EAAE,CAAC;QACtB,gBAAgB,EAAE,CAAC;KACpB,CAAA;IAED,MAAM,GAAG,GAAG,OAAO,CAAC;QAClB,KAAK,EAAE,eAAe;QACtB,KAAK,EAAE,CAAC;KACT,CAAC,CAAA;IAEF,KAAK,UAAU,YAAY;QACzB,MAAM,UAAU,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,UAAU,CAAC,QAAQ,CAAC;YACzD,KAAK,EAAE,GAAG,CACR,EAAE,CAAC,WAAW,CAAC,UAAU,CAAC,QAAQ,EAAE,MAAM,CAAC,EAAE,CAAC,EAC9C,EAAE,CAAC,WAAW,CAAC,UAAU,CAAC,IAAI,EAAE,WAAW,CAAC,EAC5C,OAAO,CAAC,WAAW,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC,EAC7D,EAAE,CAAC,WAAW,CAAC,UAAU,CAAC,EAAE,EAAE,MAAM,CAAC,CACtC;YACD,OAAO,EAAE,CAAC,GAAG,CAAC,WAAW,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;YACzC,KAAK,EAAE,SAAS;SACjB,CAAC,CAAA;QACF,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC5B,OAAO,IAAI,CAAA;QACb,CAAC;QAED,MAAM,GAAG,UAAU,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,IAAI,MAAM,CAAA;QACxC,OAAO,UAAU,CAAA;IACnB,CAAC;IAED,MAAM,YAAY,GAAG,qBAAqB,CAAC,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,CAAA;IAC9F,MAAM,KAAK,GAAG,IAAI,MAAM,CAAC,EAAE,WAAW,EAAE,CAAC,CAAA;IAEzC,OAAO,IAAI,EAAE,CAAC;QACZ,MAAM,KAAK,CAAC,cAAc,CAAC,WAAW,CAAC,CAAA;QACvC,MAAM,UAAU,GAAG,MAAM,YAAY,EAAE,CAAA;QACvC,IAAI,CAAC,UAAU;YAAE,MAAK;QACtB,WAAW,EAAE,CAAA;QACb,MAAM,kBAAkB,GAAG,WAAW,CAAA;QACtC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,YAAY,CAAC,EAAE,UAAU,EAAE,WAAW,EAAE,kBAAkB,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;IACrG,CAAC;IAED,MAAM,KAAK,CAAC,MAAM,EAAE,CAAA;IAEpB,GAAG,CAAC,OAAO,CAAC,SAAS,KAAK,CAAC,mBAAmB,YAAY,KAAK,CAAC,gBAAgB,SAAS,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAA;IAE7G,MAAM,YAAY,CAAC;QACjB,OAAO;QACP,QAAQ,EAAE,MAAM,CAAC,EAAE;QACnB,MAAM,EAAE,KAAK,CAAC,gBAAgB,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,WAAW;KAC5D,CAAC,CAAA;AACJ,CAAC"}
1
+ {"version":3,"file":"add-pieces.js","sourceRoot":"","sources":["../../../src/pipeline/add-pieces.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,gBAAgB,CAAA;AACxC,OAAO,KAAK,EAAE,MAAM,wBAAwB,CAAA;AAC5C,OAAO,EAAE,GAAG,EAAE,GAAG,EAAE,EAAE,EAAE,EAAE,EAAE,OAAO,EAAE,MAAM,aAAa,CAAA;AACvD,OAAO,MAAM,MAAM,SAAS,CAAA;AAC5B,OAAO,EAAE,iBAAiB,EAAE,MAAM,WAAW,CAAA;AAC7C,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAA;AACjD,OAAO,EAAE,gBAAgB,EAAE,MAAM,6BAA6B,CAAA;AAC9D,OAAO,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAA;AACrD,OAAO,EAAE,gBAAgB,EAAE,MAAM,4BAA4B,CAAA;AAG7D,OAAO,EACL,2BAA2B,EAC3B,sBAAsB,EACtB,iBAAiB,EACjB,QAAQ,EACR,qBAAqB,EACrB,sBAAsB,GACvB,MAAM,aAAa,CAAA;AAGpB,MAAM,oBAAoB,GAAG,GAAG,CAAA;AAkChC,SAAS,qBAAqB,CAAC,EAAE,OAAO,EAAE,UAAU,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG,EAAgC;IAC9G,OAAO,KAAK,EAAE,OAA4B,EAAE,EAAE;QAC5C,IAAI,YAAY,GAAG,CAAC,CAAA;QACpB,IAAI,SAAS,GAAG,CAAC,CAAA;QACjB,IAAI,UAAU,GAAsB,OAAO,CAAC,UAAU,CAAA;QAEtD,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC,SAAS,OAAO,CAAC,WAAW,KAAK,OAAO,CAAC,UAAU,CAAC,MAAM,UAAU,CAAC,CAAA;QAE7F,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,MAAM,gBAAgB,CAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,CAAC,EAAE,EAAE,MAAM,EAAE,CAAC,CAAA;YAGhF,IAAI,UAAU,EAAE,CAAC;gBACf,UAAU,GAAG,MAAM,UAAU,CAAC,EAAE,OAAO,EAAE,YAAY,EAAE,UAAU,EAAE,UAAU,EAAE,CAAC,CAAA;YAClF,CAAC;YAED,MAAM,iCAAiC,GAAG,UAAU,CAAC,MAAM,CAAA;YAC3D,UAAU,GAAG,MAAM,2BAA2B,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC,CAAA;YAC/E,MAAM,mBAAmB,GAAG,iCAAiC,GAAG,UAAU,CAAC,MAAM,CAAA;YACjF,IAAI,mBAAmB,GAAG,CAAC,EAAE,CAAC;gBAC5B,KAAK,CAAC,mBAAmB,IAAI,mBAAmB,CAAA;gBAChD,YAAY,IAAI,mBAAmB,CAAA;YACrC,CAAC;YAED,KAAK,CAAC,OAAO,CAAC,WAAW,UAAU,CAAC,MAAM,YAAY,CAAC,CAAA;YAEvD,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC1B,MAAM,UAAU,GAAG,MAAM,EAAE,CAAC,iBAAiB,CAAC,MAAM,EAAE;oBACpD,UAAU,EAAE,MAAM,CAAC,iBAAiB;oBACpC,SAAS,EAAE,OAAO,CAAC,SAAS;oBAC5B,eAAe,EAAE,OAAO,CAAC,eAAe;oBACxC,MAAM,EAAE,sBAAsB,CAAC,UAAU,CAAC;oBAC1C,QAAQ,EAAE,CAAC,MAAM,EAAE,EAAE;wBACnB,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,KAAK,UAAU,CAAC,CAAC,MAAM,CAAA;wBACrF,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,MAAM,CAAA;wBAChF,KAAK,CAAC,OAAO,CAAC,QAAQ,SAAS,eAAe,MAAM,SAAS,CAAC,CAAA;oBAChE,CAAC;iBACF,CAAC,CAAA;gBAEF,KAAK,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,UAAU,CAAC,MAAM,EAAE,CAAC;oBACrD,MAAM,GAAG,GAAG,QAAQ,CAAC,QAAQ,EAAE,CAAA;oBAC/B,IAAI,MAAM,KAAK,UAAU,EAAE,CAAC;wBAC1B,KAAK,CAAC,gBAAgB,EAAE,CAAA;wBACxB,SAAS,EAAE,CAAA;wBACX,MAAM,EAAE,iBAAiB,EAAE,iBAAiB,EAAE,GAAG,sBAAsB,CAAC,UAAU,EAAE,GAAG,CAAC,CAAA;wBACxF,UAAU,GAAG,iBAAiB,CAAA;wBAC9B,MAAM,gBAAgB,CAAC;4BACrB,OAAO;4BACP,UAAU,EAAE,iBAAiB,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;gCAChD,GAAG,SAAS;gCACZ,MAAM,EAAE,QAAQ;gCAChB,KAAK,EAAE,2BAA2B,MAAM,EAAE;6BAC3C,CAAC,CAAC;yBACJ,CAAC,CAAA;oBACJ,CAAC;gBACH,CAAC;YACH,CAAC;YAGD,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC1B,KAAK,CAAC,OAAO,CAAC,UAAU,UAAU,CAAC,MAAM,YAAY,CAAC,CAAA;gBACtD,MAAM,eAAe,GAAG,MAAM,EAAE,CAAC,SAAS,CAAC,MAAM,EAAE;oBACjD,UAAU,EAAE,MAAM,CAAC,iBAAiB;oBACpC,SAAS,EAAE,OAAO,CAAC,SAAS;oBAC5B,eAAe,EAAE,OAAO,CAAC,eAAe;oBACxC,MAAM,EAAE,qBAAqB,CAAC,UAAU,EAAE,CAAC,UAAU,CAAC;iBACvD,CAAC,CAAA;gBACF,MAAM,gBAAgB,CAAC;oBACrB,OAAO;oBACP,UAAU,EAAE,UAAU,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;wBACzC,GAAG,SAAS;wBACZ,MAAM,EAAE,eAAe,CAAC,MAAM;qBAC/B,CAAC,CAAC;iBACJ,CAAC,CAAA;gBAEF,KAAK,CAAC,OAAO,CAAC,0BAA0B,QAAQ,CAAC,eAAe,CAAC,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAA;gBAC5F,MAAM,gBAAgB,GAAG,MAAM,EAAE,CAAC,gBAAgB,CAAC,eAAe,CAAC,CAAA;gBAEnE,MAAM,eAAe,GAAG,gBAAgB,CAAC,eAAe,IAAI,gBAAgB,CAAC,MAAM,CAAA;gBACnF,KAAK,CAAC,mBAAmB,IAAI,UAAU,CAAC,MAAM,CAAA;gBAC9C,YAAY,IAAI,UAAU,CAAC,MAAM,CAAA;gBACjC,KAAK,MAAM,SAAS,IAAI,UAAU;oBAAE,UAAU,EAAE,GAAG,CAAC,SAAS,CAAC,GAAG,CAAC,CAAA;gBAClE,MAAM,gBAAgB,CAAC;oBACrB,OAAO;oBACP,UAAU,EAAE,UAAU,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;wBACzC,GAAG,SAAS;wBACZ,MAAM,EAAE,WAAW;wBACnB,KAAK,EAAE,IAAI;wBACX,MAAM,EAAE,eAAe;qBACxB,CAAC,CAAC;iBACJ,CAAC,CAAA;YACJ,CAAC;YACD,KAAK,CAAC,OAAO,CAAC,SAAS,YAAY,WAAW,SAAS,SAAS,CAAC,CAAA;QACnE,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,KAAK,CAAC,gBAAgB,IAAI,UAAU,CAAC,MAAM,CAAA;YAC3C,MAAM,OAAO,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,CAAA;YACxE,KAAK,CAAC,OAAO,CAAC,GAAG,OAAO,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,EAAE,CAAC,CAAA;YAC/C,MAAM,gBAAgB,CAAC;gBACrB,OAAO;gBACP,UAAU,EAAE,UAAU,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;oBACzC,GAAG,SAAS;oBACZ,MAAM,EAAE,QAAQ;oBAChB,KAAK,EAAE,OAAO;iBACf,CAAC,CAAC;aACJ,CAAC,CAAA;QACJ,CAAC;IACH,CAAC,CAAA;AACH,CAAC;AAKD,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,EACjC,OAAO,EACP,GAAG,EACH,MAAM,EACN,WAAW,EACX,SAAS,EACT,MAAM,GACoB;IAC1B,MAAM,WAAW,GAAG,OAAO,CAAC,CAAC,CAAC,UAAU,CAAA;IACxC,IAAI,MAAM,GAAG,CAAC,CAAA;IAEd,MAAM,eAAe,GAAG,MAAM,OAAO,CAAC,MAAM,CAC1C,WAAW,CAAC,UAAU,EACtB,GAAG,CACD,EAAE,CAAC,WAAW,CAAC,UAAU,CAAC,QAAQ,EAAE,MAAM,CAAC,EAAE,CAAC,EAC9C,EAAE,CAAC,WAAW,CAAC,UAAU,CAAC,IAAI,EAAE,WAAW,CAAC,EAC5C,OAAO,CAAC,WAAW,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC,CAC9D,CACF,CAAA;IACD,IAAI,WAAW,GAAG,CAAC,CAAA;IACnB,MAAM,KAAK,GAAG;QACZ,eAAe;QACf,mBAAmB,EAAE,CAAC;QACtB,gBAAgB,EAAE,CAAC;KACpB,CAAA;IAED,MAAM,GAAG,GAAG,OAAO,CAAC;QAClB,KAAK,EAAE,eAAe;QACtB,KAAK,EAAE,CAAC;KACT,CAAC,CAAA;IAEF,MAAM,OAAO,GAAG,eAAe,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,gBAAgB,CAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,CAAC,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC,SAAS,CAAA;IAGlH,IAAI,UAAmC,CAAA;IACvC,IAAI,MAAM,CAAC,eAAe,IAAI,IAAI,IAAI,OAAO,EAAE,CAAC;QAC9C,MAAM,MAAM,GAAG,MAAM,iBAAiB,CAAC,GAAG,EAAE,OAAO,CAAC,SAAS,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAA;QAClF,UAAU,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,IAAI,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAA;IACzF,CAAC;IAED,KAAK,UAAU,YAAY;QACzB,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,OAAO,IAAI,CAAA;QACb,CAAC;QACD,MAAM,UAAU,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,UAAU,CAAC,QAAQ,CAAC;YACzD,KAAK,EAAE,GAAG,CACR,EAAE,CAAC,WAAW,CAAC,UAAU,CAAC,QAAQ,EAAE,MAAM,CAAC,EAAE,CAAC,EAC9C,EAAE,CAAC,WAAW,CAAC,UAAU,CAAC,IAAI,EAAE,WAAW,CAAC,EAC5C,OAAO,CAAC,WAAW,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC,EAC7D,EAAE,CAAC,WAAW,CAAC,UAAU,CAAC,EAAE,EAAE,MAAM,CAAC,CACtC;YACD,OAAO,EAAE,CAAC,GAAG,CAAC,WAAW,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;YACzC,KAAK,EAAE,SAAS,IAAI,oBAAoB;SACzC,CAAC,CAAA;QACF,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC5B,OAAO,IAAI,CAAA;QACb,CAAC;QAGD,MAAM,UAAU,GAAG,iBAAiB,CAAC;YACnC,UAAU,EAAE,UAAU;YACtB,SAAS,EAAE,OAAO,CAAC,SAAS;YAC5B,KAAK,EAAE,MAAM,CAAC,KAAK;YACnB,YAAY,EAAE,CAAC,UAAU;SAC1B,CAAC,CAAA;QAEF,MAAM,GAAG,UAAU,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,IAAI,MAAM,CAAA;QACxC,OAAO,UAAU,CAAA;IACnB,CAAC;IAED,MAAM,YAAY,GAAG,qBAAqB,CAAC,EAAE,OAAO,EAAE,UAAU,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,CAAA;IAC/F,MAAM,KAAK,GAAG,IAAI,MAAM,CAAC,EAAE,WAAW,EAAE,CAAC,CAAA;IAEzC,OAAO,IAAI,EAAE,CAAC;QACZ,MAAM,KAAK,CAAC,cAAc,CAAC,WAAW,CAAC,CAAA;QACvC,MAAM,UAAU,GAAG,MAAM,YAAY,EAAE,CAAA;QACvC,IAAI,CAAC,UAAU;YAAE,MAAK;QACtB,WAAW,EAAE,CAAA;QACb,MAAM,kBAAkB,GAAG,WAAW,CAAA;QACtC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,YAAY,CAAC,EAAE,UAAU,EAAE,WAAW,EAAE,kBAAkB,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;IACrG,CAAC;IAED,MAAM,KAAK,CAAC,MAAM,EAAE,CAAA;IAEpB,GAAG,CAAC,OAAO,CAAC,SAAS,KAAK,CAAC,mBAAmB,YAAY,KAAK,CAAC,gBAAgB,SAAS,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAA;IAE7G,MAAM,YAAY,CAAC;QACjB,OAAO;QACP,QAAQ,EAAE,MAAM,CAAC,EAAE;QACnB,MAAM,EAAE,KAAK,CAAC,gBAAgB,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,WAAW;KAC5D,CAAC,CAAA;AACJ,CAAC"}
@@ -1,14 +1,14 @@
1
1
  import type { Address } from 'viem';
2
2
  import type { RepairSelect } from '../local-schema.ts';
3
- import type { IndexerDatabase, LocalDatabase, WalletClient } from '../types.ts';
3
+ import type { IndexerApi, LocalDatabase, WalletClient } from '../types.ts';
4
4
  export type EnsureRepairDatasetOptions = {
5
5
  source: string;
6
6
  localDb: LocalDatabase;
7
- indexerDb: IndexerDatabase;
7
+ api: IndexerApi;
8
8
  client: WalletClient;
9
9
  repair: RepairSelect;
10
10
  payer: Address;
11
11
  };
12
- export declare function ensureRepairDataset({ source, localDb, indexerDb, client, repair, payer, }: EnsureRepairDatasetOptions): Promise<bigint>;
13
- export declare function ensureReplicateDataset({ localDb, indexerDb, client, repair, payer, }: EnsureRepairDatasetOptions): Promise<bigint>;
12
+ export declare function ensureRepairDataset({ source, localDb, api, client, repair, payer }: EnsureRepairDatasetOptions): Promise<bigint>;
13
+ export declare function ensureReplicateDataset({ localDb, api, client, repair, payer }: EnsureRepairDatasetOptions): Promise<bigint>;
14
14
  //# sourceMappingURL=create-datasets.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"create-datasets.d.ts","sourceRoot":"","sources":["../../../src/pipeline/create-datasets.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,MAAM,CAAA;AAGnC,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAA;AACtD,OAAO,KAAK,EAAE,eAAe,EAAE,aAAa,EAAE,YAAY,EAAE,MAAM,aAAa,CAAA;AAG/E,MAAM,MAAM,0BAA0B,GAAG;IACvC,MAAM,EAAE,MAAM,CAAA;IACd,OAAO,EAAE,aAAa,CAAA;IACtB,SAAS,EAAE,eAAe,CAAA;IAC1B,MAAM,EAAE,YAAY,CAAA;IACpB,MAAM,EAAE,YAAY,CAAA;IACpB,KAAK,EAAE,OAAO,CAAA;CACf,CAAA;AAOD,wBAAsB,mBAAmB,CAAC,EACxC,MAAM,EACN,OAAO,EACP,SAAS,EACT,MAAM,EACN,MAAM,EACN,KAAK,GACN,EAAE,0BAA0B,mBA8C5B;AAOD,wBAAsB,sBAAsB,CAAC,EAC3C,OAAO,EACP,SAAS,EACT,MAAM,EACN,MAAM,EACN,KAAK,GACN,EAAE,0BAA0B,mBAqD5B"}
1
+ {"version":3,"file":"create-datasets.d.ts","sourceRoot":"","sources":["../../../src/pipeline/create-datasets.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,MAAM,CAAA;AAInC,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAA;AACtD,OAAO,KAAK,EAAE,UAAU,EAAE,aAAa,EAAE,YAAY,EAAE,MAAM,aAAa,CAAA;AAG1E,MAAM,MAAM,0BAA0B,GAAG;IACvC,MAAM,EAAE,MAAM,CAAA;IACd,OAAO,EAAE,aAAa,CAAA;IACtB,GAAG,EAAE,UAAU,CAAA;IACf,MAAM,EAAE,YAAY,CAAA;IACpB,MAAM,EAAE,YAAY,CAAA;IACpB,KAAK,EAAE,OAAO,CAAA;CACf,CAAA;AAOD,wBAAsB,mBAAmB,CAAC,EAAE,MAAM,EAAE,OAAO,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,EAAE,0BAA0B,mBA8CpH;AAOD,wBAAsB,sBAAsB,CAAC,EAAE,OAAO,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,EAAE,0BAA0B,mBA8C/G"}
@@ -1,11 +1,11 @@
1
1
  import * as p from '@clack/prompts';
2
2
  import * as SP from '@filoz/synapse-core/sp';
3
3
  import { getPDPProvider } from '@filoz/synapse-core/sp-registry';
4
- import { eq } from 'drizzle-orm';
4
+ import { getDataSet, toStringMetadata } from "../api.js";
5
5
  import { findRepairDataset } from "../db/find-repair-dataset.js";
6
6
  import { repairUpdate } from "../db/repair-update.js";
7
7
  import { hashLink } from "../utils.js";
8
- export async function ensureRepairDataset({ source, localDb, indexerDb, client, repair, payer, }) {
8
+ export async function ensureRepairDataset({ source, localDb, api, client, repair, payer }) {
9
9
  const log = p.taskLog({
10
10
  title: 'Ensuring repair dataset',
11
11
  });
@@ -16,7 +16,7 @@ export async function ensureRepairDataset({ source, localDb, indexerDb, client,
16
16
  throw new Error(`Target provider ${repair.targetProviderId} not found or inactive`);
17
17
  let datasetId = null;
18
18
  const existingDatasetId = await findRepairDataset({
19
- indexerDb,
19
+ api,
20
20
  providerId: repair.targetProviderId,
21
21
  payer,
22
22
  source,
@@ -50,7 +50,7 @@ export async function ensureRepairDataset({ source, localDb, indexerDb, client,
50
50
  });
51
51
  return datasetId;
52
52
  }
53
- export async function ensureReplicateDataset({ localDb, indexerDb, client, repair, payer, }) {
53
+ export async function ensureReplicateDataset({ localDb, api, client, repair, payer }) {
54
54
  const log = p.taskLog({
55
55
  title: 'Ensuring replication dataset',
56
56
  });
@@ -66,14 +66,7 @@ export async function ensureReplicateDataset({ localDb, indexerDb, client, repai
66
66
  log.success(`Data set #${repair.targetDataSetId} already exists at ${provider.pdp.serviceURL}`);
67
67
  return repair.targetDataSetId;
68
68
  }
69
- const schema = indexerDb._.fullSchema;
70
- const sourceDataSet = await indexerDb.query.dataSets.findFirst({
71
- where: eq(schema.dataSets.dataSetId, repair.repairDataSetId),
72
- columns: {
73
- metadata: true,
74
- withCdn: true,
75
- },
76
- });
69
+ const sourceDataSet = await getDataSet(api, repair.repairDataSetId);
77
70
  if (!sourceDataSet) {
78
71
  throw new Error(`Source dataset ${repair.repairDataSetId} not found`);
79
72
  }
@@ -81,8 +74,8 @@ export async function ensureReplicateDataset({ localDb, indexerDb, client, repai
81
74
  payee: provider.payee,
82
75
  serviceURL: provider.pdp.serviceURL,
83
76
  payer,
84
- cdn: sourceDataSet.withCdn,
85
- metadata: sourceDataSet.metadata ?? undefined,
77
+ cdn: sourceDataSet.withCdn ?? false,
78
+ metadata: sourceDataSet.metadata ? toStringMetadata(sourceDataSet.metadata) : undefined,
86
79
  });
87
80
  log.message(`Waiting for data to be created at ${provider.pdp.serviceURL} ${hashLink(txHash, client.chain)}...`);
88
81
  const waitForResult = await SP.waitForCreateDataSet({
@@ -1 +1 @@
1
- {"version":3,"file":"create-datasets.js","sourceRoot":"","sources":["../../../src/pipeline/create-datasets.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,CAAC,MAAM,gBAAgB,CAAA;AACnC,OAAO,KAAK,EAAE,MAAM,wBAAwB,CAAA;AAC5C,OAAO,EAAE,cAAc,EAAE,MAAM,iCAAiC,CAAA;AAChE,OAAO,EAAE,EAAE,EAAE,MAAM,aAAa,CAAA;AAEhC,OAAO,EAAE,iBAAiB,EAAE,MAAM,8BAA8B,CAAA;AAChE,OAAO,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAA;AAGrD,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAA;AAgBtC,MAAM,CAAC,KAAK,UAAU,mBAAmB,CAAC,EACxC,MAAM,EACN,OAAO,EACP,SAAS,EACT,MAAM,EACN,MAAM,EACN,KAAK,GACsB;IAC3B,MAAM,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC;QACpB,KAAK,EAAE,yBAAyB;KACjC,CAAC,CAAA;IACF,MAAM,QAAQ,GAAG,MAAM,cAAc,CAAC,MAAM,EAAE;QAC5C,UAAU,EAAE,MAAM,CAAC,gBAAgB;KACpC,CAAC,CAAA;IAEF,IAAI,CAAC,QAAQ;QAAE,MAAM,IAAI,KAAK,CAAC,mBAAmB,MAAM,CAAC,gBAAgB,wBAAwB,CAAC,CAAA;IAElG,IAAI,SAAS,GAAkB,IAAI,CAAA;IAEnC,MAAM,iBAAiB,GAAG,MAAM,iBAAiB,CAAC;QAChD,SAAS;QACT,UAAU,EAAE,MAAM,CAAC,gBAAgB;QACnC,KAAK;QACL,MAAM;KACP,CAAC,CAAA;IAEF,IAAI,iBAAiB,EAAE,CAAC;QACtB,SAAS,GAAG,iBAAiB,CAAA;QAC7B,GAAG,CAAC,OAAO,CAAC,aAAa,SAAS,sBAAsB,QAAQ,CAAC,GAAG,CAAC,UAAU,EAAE,CAAC,CAAA;IACpF,CAAC;SAAM,CAAC;QACN,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,GAAG,MAAM,EAAE,CAAC,aAAa,CAAC,MAAM,EAAE;YAC3D,KAAK,EAAE,QAAQ,CAAC,KAAK;YACrB,UAAU,EAAE,QAAQ,CAAC,GAAG,CAAC,UAAU;YACnC,KAAK;YACL,GAAG,EAAE,KAAK;YACV,QAAQ,EAAE;gBACR,MAAM;gBACN,gBAAgB,EAAE,EAAE;aACrB;SACF,CAAC,CAAA;QACF,GAAG,CAAC,OAAO,CAAC,qCAAqC,QAAQ,CAAC,GAAG,CAAC,UAAU,IAAI,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAA;QAChH,MAAM,aAAa,GAAG,MAAM,EAAE,CAAC,oBAAoB,CAAC;YAClD,SAAS;SACV,CAAC,CAAA;QACF,SAAS,GAAG,aAAa,CAAC,SAAS,CAAA;QACnC,GAAG,CAAC,OAAO,CAAC,aAAa,SAAS,eAAe,QAAQ,CAAC,GAAG,CAAC,UAAU,EAAE,CAAC,CAAA;IAC7E,CAAC;IACD,MAAM,YAAY,CAAC;QACjB,OAAO;QACP,QAAQ,EAAE,MAAM,CAAC,EAAE;QACnB,eAAe,EAAE,SAAS;KAC3B,CAAC,CAAA;IACF,OAAO,SAAS,CAAA;AAClB,CAAC;AAOD,MAAM,CAAC,KAAK,UAAU,sBAAsB,CAAC,EAC3C,OAAO,EACP,SAAS,EACT,MAAM,EACN,MAAM,EACN,KAAK,GACsB;IAC3B,MAAM,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC;QACpB,KAAK,EAAE,8BAA8B;KACtC,CAAC,CAAA;IAEF,IAAI,MAAM,CAAC,eAAe,IAAI,IAAI,EAAE,CAAC;QACnC,MAAM,IAAI,KAAK,CAAC,2BAA2B,CAAC,CAAA;IAC9C,CAAC;IAED,MAAM,QAAQ,GAAG,MAAM,cAAc,CAAC,MAAM,EAAE;QAC5C,UAAU,EAAE,MAAM,CAAC,gBAAgB;KACpC,CAAC,CAAA;IAEF,IAAI,CAAC,QAAQ;QAAE,MAAM,IAAI,KAAK,CAAC,mBAAmB,MAAM,CAAC,gBAAgB,wBAAwB,CAAC,CAAA;IAElG,IAAI,MAAM,CAAC,eAAe,IAAI,IAAI,EAAE,CAAC;QACnC,GAAG,CAAC,OAAO,CAAC,aAAa,MAAM,CAAC,eAAe,sBAAsB,QAAQ,CAAC,GAAG,CAAC,UAAU,EAAE,CAAC,CAAA;QAC/F,OAAO,MAAM,CAAC,eAAe,CAAA;IAC/B,CAAC;IAED,MAAM,MAAM,GAAG,SAAS,CAAC,CAAC,CAAC,UAAU,CAAA;IACrC,MAAM,aAAa,GAAG,MAAM,SAAS,CAAC,KAAK,CAAC,QAAQ,CAAC,SAAS,CAAC;QAC7D,KAAK,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC,eAAe,CAAC;QAC5D,OAAO,EAAE;YACP,QAAQ,EAAE,IAAI;YACd,OAAO,EAAE,IAAI;SACd;KACF,CAAC,CAAA;IAEF,IAAI,CAAC,aAAa,EAAE,CAAC;QACnB,MAAM,IAAI,KAAK,CAAC,kBAAkB,MAAM,CAAC,eAAe,YAAY,CAAC,CAAA;IACvE,CAAC;IAED,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,GAAG,MAAM,EAAE,CAAC,aAAa,CAAC,MAAM,EAAE;QAC3D,KAAK,EAAE,QAAQ,CAAC,KAAK;QACrB,UAAU,EAAE,QAAQ,CAAC,GAAG,CAAC,UAAU;QACnC,KAAK;QACL,GAAG,EAAE,aAAa,CAAC,OAAO;QAC1B,QAAQ,EAAE,aAAa,CAAC,QAAQ,IAAI,SAAS;KAC9C,CAAC,CAAA;IACF,GAAG,CAAC,OAAO,CAAC,qCAAqC,QAAQ,CAAC,GAAG,CAAC,UAAU,IAAI,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAA;IAChH,MAAM,aAAa,GAAG,MAAM,EAAE,CAAC,oBAAoB,CAAC;QAClD,SAAS;KACV,CAAC,CAAA;IACF,MAAM,SAAS,GAAG,aAAa,CAAC,SAAS,CAAA;IACzC,GAAG,CAAC,OAAO,CAAC,aAAa,SAAS,eAAe,QAAQ,CAAC,GAAG,CAAC,UAAU,EAAE,CAAC,CAAA;IAE3E,MAAM,YAAY,CAAC;QACjB,OAAO;QACP,QAAQ,EAAE,MAAM,CAAC,EAAE;QACnB,eAAe,EAAE,SAAS;KAC3B,CAAC,CAAA;IACF,OAAO,SAAS,CAAA;AAClB,CAAC"}
1
+ {"version":3,"file":"create-datasets.js","sourceRoot":"","sources":["../../../src/pipeline/create-datasets.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,CAAC,MAAM,gBAAgB,CAAA;AACnC,OAAO,KAAK,EAAE,MAAM,wBAAwB,CAAA;AAC5C,OAAO,EAAE,cAAc,EAAE,MAAM,iCAAiC,CAAA;AAEhE,OAAO,EAAE,UAAU,EAAE,gBAAgB,EAAE,MAAM,WAAW,CAAA;AACxD,OAAO,EAAE,iBAAiB,EAAE,MAAM,8BAA8B,CAAA;AAChE,OAAO,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAA;AAGrD,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAA;AAgBtC,MAAM,CAAC,KAAK,UAAU,mBAAmB,CAAC,EAAE,MAAM,EAAE,OAAO,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAA8B;IACnH,MAAM,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC;QACpB,KAAK,EAAE,yBAAyB;KACjC,CAAC,CAAA;IACF,MAAM,QAAQ,GAAG,MAAM,cAAc,CAAC,MAAM,EAAE;QAC5C,UAAU,EAAE,MAAM,CAAC,gBAAgB;KACpC,CAAC,CAAA;IAEF,IAAI,CAAC,QAAQ;QAAE,MAAM,IAAI,KAAK,CAAC,mBAAmB,MAAM,CAAC,gBAAgB,wBAAwB,CAAC,CAAA;IAElG,IAAI,SAAS,GAAkB,IAAI,CAAA;IAEnC,MAAM,iBAAiB,GAAG,MAAM,iBAAiB,CAAC;QAChD,GAAG;QACH,UAAU,EAAE,MAAM,CAAC,gBAAgB;QACnC,KAAK;QACL,MAAM;KACP,CAAC,CAAA;IAEF,IAAI,iBAAiB,EAAE,CAAC;QACtB,SAAS,GAAG,iBAAiB,CAAA;QAC7B,GAAG,CAAC,OAAO,CAAC,aAAa,SAAS,sBAAsB,QAAQ,CAAC,GAAG,CAAC,UAAU,EAAE,CAAC,CAAA;IACpF,CAAC;SAAM,CAAC;QACN,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,GAAG,MAAM,EAAE,CAAC,aAAa,CAAC,MAAM,EAAE;YAC3D,KAAK,EAAE,QAAQ,CAAC,KAAK;YACrB,UAAU,EAAE,QAAQ,CAAC,GAAG,CAAC,UAAU;YACnC,KAAK;YACL,GAAG,EAAE,KAAK;YACV,QAAQ,EAAE;gBACR,MAAM;gBACN,gBAAgB,EAAE,EAAE;aACrB;SACF,CAAC,CAAA;QACF,GAAG,CAAC,OAAO,CAAC,qCAAqC,QAAQ,CAAC,GAAG,CAAC,UAAU,IAAI,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAA;QAChH,MAAM,aAAa,GAAG,MAAM,EAAE,CAAC,oBAAoB,CAAC;YAClD,SAAS;SACV,CAAC,CAAA;QACF,SAAS,GAAG,aAAa,CAAC,SAAS,CAAA;QACnC,GAAG,CAAC,OAAO,CAAC,aAAa,SAAS,eAAe,QAAQ,CAAC,GAAG,CAAC,UAAU,EAAE,CAAC,CAAA;IAC7E,CAAC;IACD,MAAM,YAAY,CAAC;QACjB,OAAO;QACP,QAAQ,EAAE,MAAM,CAAC,EAAE;QACnB,eAAe,EAAE,SAAS;KAC3B,CAAC,CAAA;IACF,OAAO,SAAS,CAAA;AAClB,CAAC;AAOD,MAAM,CAAC,KAAK,UAAU,sBAAsB,CAAC,EAAE,OAAO,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAA8B;IAC9G,MAAM,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC;QACpB,KAAK,EAAE,8BAA8B;KACtC,CAAC,CAAA;IAEF,IAAI,MAAM,CAAC,eAAe,IAAI,IAAI,EAAE,CAAC;QACnC,MAAM,IAAI,KAAK,CAAC,2BAA2B,CAAC,CAAA;IAC9C,CAAC;IAED,MAAM,QAAQ,GAAG,MAAM,cAAc,CAAC,MAAM,EAAE;QAC5C,UAAU,EAAE,MAAM,CAAC,gBAAgB;KACpC,CAAC,CAAA;IAEF,IAAI,CAAC,QAAQ;QAAE,MAAM,IAAI,KAAK,CAAC,mBAAmB,MAAM,CAAC,gBAAgB,wBAAwB,CAAC,CAAA;IAElG,IAAI,MAAM,CAAC,eAAe,IAAI,IAAI,EAAE,CAAC;QACnC,GAAG,CAAC,OAAO,CAAC,aAAa,MAAM,CAAC,eAAe,sBAAsB,QAAQ,CAAC,GAAG,CAAC,UAAU,EAAE,CAAC,CAAA;QAC/F,OAAO,MAAM,CAAC,eAAe,CAAA;IAC/B,CAAC;IAED,MAAM,aAAa,GAAG,MAAM,UAAU,CAAC,GAAG,EAAE,MAAM,CAAC,eAAe,CAAC,CAAA;IAEnE,IAAI,CAAC,aAAa,EAAE,CAAC;QACnB,MAAM,IAAI,KAAK,CAAC,kBAAkB,MAAM,CAAC,eAAe,YAAY,CAAC,CAAA;IACvE,CAAC;IAED,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,GAAG,MAAM,EAAE,CAAC,aAAa,CAAC,MAAM,EAAE;QAC3D,KAAK,EAAE,QAAQ,CAAC,KAAK;QACrB,UAAU,EAAE,QAAQ,CAAC,GAAG,CAAC,UAAU;QACnC,KAAK;QACL,GAAG,EAAE,aAAa,CAAC,OAAO,IAAI,KAAK;QACnC,QAAQ,EAAE,aAAa,CAAC,QAAQ,CAAC,CAAC,CAAC,gBAAgB,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,SAAS;KACxF,CAAC,CAAA;IACF,GAAG,CAAC,OAAO,CAAC,qCAAqC,QAAQ,CAAC,GAAG,CAAC,UAAU,IAAI,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAA;IAChH,MAAM,aAAa,GAAG,MAAM,EAAE,CAAC,oBAAoB,CAAC;QAClD,SAAS;KACV,CAAC,CAAA;IACF,MAAM,SAAS,GAAG,aAAa,CAAC,SAAS,CAAA;IACzC,GAAG,CAAC,OAAO,CAAC,aAAa,SAAS,eAAe,QAAQ,CAAC,GAAG,CAAC,UAAU,EAAE,CAAC,CAAA;IAE3E,MAAM,YAAY,CAAC;QACjB,OAAO;QACP,QAAQ,EAAE,MAAM,CAAC,EAAE;QACnB,eAAe,EAAE,SAAS;KAC3B,CAAC,CAAA;IACF,OAAO,SAAS,CAAA;AAClB,CAAC"}
@@ -1,27 +1,26 @@
1
- import type { Chain } from '@filoz/synapse-core/chains';
1
+ import type { ApiClient } from '@filecoin-project/api-client';
2
+ import type { FilecoinChain } from '@filoz/synapse-core/chains';
2
3
  import type { Client } from '@libsql/client';
3
4
  import type { LibSQLDatabase } from 'drizzle-orm/libsql';
4
- import type { NodePgDatabase } from 'drizzle-orm/node-postgres';
5
5
  import type { z } from 'incur';
6
- import type { Pool } from 'pg';
7
6
  import type { Account, Address, Hex, Transport, Client as ViemClient } from 'viem';
8
- import type * as indexerSchema from './indexer-schema.ts';
7
+ import type { Network } from './api.ts';
9
8
  import type * as localSchema from './local-schema.ts';
10
9
  import type { contextSchema } from './middleware.ts';
11
10
  export type LocalDatabase = LibSQLDatabase<typeof localSchema> & {
12
11
  $client: Client;
13
12
  };
14
- export type IndexerDatabase = NodePgDatabase<typeof indexerSchema> & {
15
- $client: Pool;
13
+ export type IndexerApi = {
14
+ client: ApiClient;
15
+ network: Network;
16
16
  };
17
17
  export interface Config {
18
18
  privateKey: Hex;
19
- indexerMainnetUrl: string;
20
- indexerCalibrationUrl: string;
19
+ apiUrl: string;
21
20
  chainId: number;
22
21
  dbPath: string;
23
22
  }
24
- export type WalletClient = ViemClient<Transport, Chain, Account>;
23
+ export type WalletClient = ViemClient<Transport, FilecoinChain, Account>;
25
24
  export type Context = z.infer<typeof contextSchema>;
26
25
  export type RepairProvider = {
27
26
  providerId: bigint;
@@ -1 +1 @@
1
- {"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,4BAA4B,CAAA;AACvD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,gBAAgB,CAAA;AAC5C,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAA;AACxD,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,2BAA2B,CAAA;AAC/D,OAAO,KAAK,EAAE,CAAC,EAAE,MAAM,OAAO,CAAA;AAC9B,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,IAAI,CAAA;AAC9B,OAAO,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,GAAG,EAAE,SAAS,EAAE,MAAM,IAAI,UAAU,EAAE,MAAM,MAAM,CAAA;AAClF,OAAO,KAAK,KAAK,aAAa,MAAM,qBAAqB,CAAA;AACzD,OAAO,KAAK,KAAK,WAAW,MAAM,mBAAmB,CAAA;AACrD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAA;AACpD,MAAM,MAAM,aAAa,GAAG,cAAc,CAAC,OAAO,WAAW,CAAC,GAAG;IAC/D,OAAO,EAAE,MAAM,CAAA;CAChB,CAAA;AAED,MAAM,MAAM,eAAe,GAAG,cAAc,CAAC,OAAO,aAAa,CAAC,GAAG;IACnE,OAAO,EAAE,IAAI,CAAA;CACd,CAAA;AAED,MAAM,WAAW,MAAM;IACrB,UAAU,EAAE,GAAG,CAAA;IACf,iBAAiB,EAAE,MAAM,CAAA;IACzB,qBAAqB,EAAE,MAAM,CAAA;IAC7B,OAAO,EAAE,MAAM,CAAA;IACf,MAAM,EAAE,MAAM,CAAA;CACf;AAED,MAAM,MAAM,YAAY,GAAG,UAAU,CAAC,SAAS,EAAE,KAAK,EAAE,OAAO,CAAC,CAAA;AAEhE,MAAM,MAAM,OAAO,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,aAAa,CAAC,CAAA;AAKnD,MAAM,MAAM,cAAc,GAAG;IAC3B,UAAU,EAAE,MAAM,CAAA;IAClB,eAAe,EAAE,OAAO,CAAA;IACxB,IAAI,EAAE,MAAM,CAAA;IACZ,UAAU,EAAE,MAAM,CAAA;IAClB,QAAQ,EAAE,OAAO,CAAA;IACjB,QAAQ,EAAE,OAAO,CAAA;CAClB,CAAA"}
1
+ {"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,8BAA8B,CAAA;AAC7D,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAA;AAC/D,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,gBAAgB,CAAA;AAC5C,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAA;AACxD,OAAO,KAAK,EAAE,CAAC,EAAE,MAAM,OAAO,CAAA;AAC9B,OAAO,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,GAAG,EAAE,SAAS,EAAE,MAAM,IAAI,UAAU,EAAE,MAAM,MAAM,CAAA;AAClF,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,UAAU,CAAA;AACvC,OAAO,KAAK,KAAK,WAAW,MAAM,mBAAmB,CAAA;AACrD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAA;AACpD,MAAM,MAAM,aAAa,GAAG,cAAc,CAAC,OAAO,WAAW,CAAC,GAAG;IAC/D,OAAO,EAAE,MAAM,CAAA;CAChB,CAAA;AAKD,MAAM,MAAM,UAAU,GAAG;IACvB,MAAM,EAAE,SAAS,CAAA;IACjB,OAAO,EAAE,OAAO,CAAA;CACjB,CAAA;AAED,MAAM,WAAW,MAAM;IACrB,UAAU,EAAE,GAAG,CAAA;IACf,MAAM,EAAE,MAAM,CAAA;IACd,OAAO,EAAE,MAAM,CAAA;IACf,MAAM,EAAE,MAAM,CAAA;CACf;AAED,MAAM,MAAM,YAAY,GAAG,UAAU,CAAC,SAAS,EAAE,aAAa,EAAE,OAAO,CAAC,CAAA;AAExE,MAAM,MAAM,OAAO,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,aAAa,CAAC,CAAA;AAKnD,MAAM,MAAM,cAAc,GAAG;IAC3B,UAAU,EAAE,MAAM,CAAA;IAClB,eAAe,EAAE,OAAO,CAAA;IACxB,IAAI,EAAE,MAAM,CAAA;IACZ,UAAU,EAAE,MAAM,CAAA;IAClB,QAAQ,EAAE,OAAO,CAAA;IACjB,QAAQ,EAAE,OAAO,CAAA;CAClB,CAAA"}