@filoz/repair-cli 1.1.0 → 1.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/package.json +1 -1
- package/dist/src/pipeline/add-pieces.d.ts +12 -0
- package/dist/src/pipeline/add-pieces.d.ts.map +1 -1
- package/dist/src/pipeline/add-pieces.js +65 -9
- package/dist/src/pipeline/add-pieces.js.map +1 -1
- package/package.json +1 -1
- package/readme.md +3 -3
- package/src/pipeline/add-pieces.ts +121 -10
package/dist/package.json
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import * as SP from '@filoz/synapse-core/sp';
|
|
1
2
|
import type { RepairSelect } from '../local-schema.ts';
|
|
2
3
|
import type { IndexerApi, LocalDatabase, WalletClient } from '../types.ts';
|
|
3
4
|
export type RunPullPiecesPhaseOptions = {
|
|
@@ -8,5 +9,16 @@ export type RunPullPiecesPhaseOptions = {
|
|
|
8
9
|
batchSize?: number;
|
|
9
10
|
client: WalletClient;
|
|
10
11
|
};
|
|
12
|
+
export declare function pullInChunks({ client, serviceURL, dataSetId, clientDataSetId, pieces: pullPieces, pullChunk, onMessage, }: {
|
|
13
|
+
client: WalletClient;
|
|
14
|
+
serviceURL: string;
|
|
15
|
+
dataSetId: bigint;
|
|
16
|
+
clientDataSetId: bigint;
|
|
17
|
+
pieces: SP.PullPieceInput[];
|
|
18
|
+
pullChunk: {
|
|
19
|
+
size: number;
|
|
20
|
+
};
|
|
21
|
+
onMessage: (message: string) => void;
|
|
22
|
+
}): Promise<SP.pullPieces.ReturnType['pieces']>;
|
|
11
23
|
export declare function runAddPieces({ localDb, api, repair, concurrency, batchSize, client, }: RunPullPiecesPhaseOptions): Promise<void>;
|
|
12
24
|
//# sourceMappingURL=add-pieces.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"add-pieces.d.ts","sourceRoot":"","sources":["../../../src/pipeline/add-pieces.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"add-pieces.d.ts","sourceRoot":"","sources":["../../../src/pipeline/add-pieces.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,MAAM,wBAAwB,CAAA;AAS5C,OAAO,KAAK,EAAmB,YAAY,EAAE,MAAM,oBAAoB,CAAA;AACvE,OAAO,KAAK,EAAE,UAAU,EAAE,aAAa,EAAE,YAAY,EAAE,MAAM,aAAa,CAAA;AAkC1E,MAAM,MAAM,yBAAyB,GAAG;IACtC,OAAO,EAAE,aAAa,CAAA;IACtB,GAAG,EAAE,UAAU,CAAA;IACf,MAAM,EAAE,YAAY,CAAA;IACpB,WAAW,EAAE,MAAM,CAAA;IAEnB,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,MAAM,EAAE,YAAY,CAAA;CACrB,CAAA;AA2BD,wBAAsB,YAAY,CAAC,EACjC,MAAM,EACN,UAAU,EACV,SAAS,EACT,eAAe,EACf,MAAM,EAAE,UAAU,EAClB,SAAS,EACT,SAAS,GACV,EAAE;IACD,MAAM,EAAE,YAAY,CAAA;IACpB,UAAU,EAAE,MAAM,CAAA;IAClB,SAAS,EAAE,MAAM,CAAA;IACjB,eAAe,EAAE,MAAM,CAAA;IACvB,MAAM,EAAE,EAAE,CAAC,cAAc,EAAE,CAAA;IAC3B,SAAS,EAAE;QAAE,IAAI,EAAE,MAAM,CAAA;KAAE,CAAA;IAC3B,SAAS,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,CAAA;CACrC,GAAG,OAAO,CAAC,EAAE,CAAC,UAAU,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,CAwD9C;AA2HD,wBAAsB,YAAY,CAAC,EACjC,OAAO,EACP,GAAG,EACH,MAAM,EACN,WAAW,EACX,SAAS,EACT,MAAM,GACP,EAAE,yBAAyB,GAAG,OAAO,CAAC,IAAI,CAAC,CAsF3C"}
|
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
import { taskLog } from '@clack/prompts';
|
|
2
|
+
import { PullError } from '@filoz/synapse-core/errors';
|
|
2
3
|
import * as SP from '@filoz/synapse-core/sp';
|
|
4
|
+
import { signAddPieces } from '@filoz/synapse-core/typed-data';
|
|
3
5
|
import { and, asc, eq, gt, inArray } from 'drizzle-orm';
|
|
4
6
|
import PQueue from 'p-queue';
|
|
5
7
|
import { listDataSetPieces } from "../api.js";
|
|
@@ -9,7 +11,62 @@ import { repairUpdate } from "../db/repair-update.js";
|
|
|
9
11
|
import { upsertOperations } from "../db/upsert-operations.js";
|
|
10
12
|
import { completeConfirmedOperations, excludeOperationsByCid, fitAddPiecesBatch, hashLink, operationsToAddPieces, operationsToPullPieces, } from "../utils.js";
|
|
11
13
|
const MAX_BATCH_CANDIDATES = 500;
|
|
12
|
-
|
|
14
|
+
const MAX_PULL_PIECES = 108;
|
|
15
|
+
const MIN_PULL_PIECES = 10;
|
|
16
|
+
const PULL_RETRY_COUNT = 10;
|
|
17
|
+
function isPullBackpressure(error) {
|
|
18
|
+
return PullError.is(error) && error.details?.includes('pull queue backpressure') === true;
|
|
19
|
+
}
|
|
20
|
+
export async function pullInChunks({ client, serviceURL, dataSetId, clientDataSetId, pieces: pullPieces, pullChunk, onMessage, }) {
|
|
21
|
+
const results = [];
|
|
22
|
+
let offset = 0;
|
|
23
|
+
while (offset < pullPieces.length) {
|
|
24
|
+
const pieces = pullPieces.slice(offset, offset + pullChunk.size);
|
|
25
|
+
const extraData = await signAddPieces(client, {
|
|
26
|
+
clientDataSetId,
|
|
27
|
+
pieces: pieces.map(({ pieceCid }) => ({ pieceCid })),
|
|
28
|
+
});
|
|
29
|
+
if (pieces.length > pullChunk.size) {
|
|
30
|
+
continue;
|
|
31
|
+
}
|
|
32
|
+
if (pieces.length > MIN_PULL_PIECES) {
|
|
33
|
+
try {
|
|
34
|
+
await SP.pullPiecesApiRequest({
|
|
35
|
+
serviceURL,
|
|
36
|
+
recordKeeper: client.chain.contracts.fwss.address,
|
|
37
|
+
extraData,
|
|
38
|
+
dataSetId,
|
|
39
|
+
pieces: pieces.map(({ pieceCid, sourceUrl }) => ({ pieceCid: pieceCid.toString(), sourceUrl })),
|
|
40
|
+
retryCount: 0,
|
|
41
|
+
});
|
|
42
|
+
}
|
|
43
|
+
catch (error) {
|
|
44
|
+
if (isPullBackpressure(error)) {
|
|
45
|
+
pullChunk.size = Math.max(MIN_PULL_PIECES, Math.floor(Math.min(pullChunk.size, pieces.length) / 2));
|
|
46
|
+
onMessage(`Curio pull queue is busy, pulling ${pullChunk.size} pieces at a time`);
|
|
47
|
+
continue;
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
const pullResult = await SP.waitForPullPieces(client, {
|
|
52
|
+
serviceURL,
|
|
53
|
+
dataSetId,
|
|
54
|
+
clientDataSetId,
|
|
55
|
+
pieces,
|
|
56
|
+
extraData,
|
|
57
|
+
retryCount: PULL_RETRY_COUNT,
|
|
58
|
+
onStatus: (status) => {
|
|
59
|
+
const completed = status.pieces.filter((piece) => piece.status === 'complete').length;
|
|
60
|
+
const failed = status.pieces.filter((piece) => piece.status === 'failed').length;
|
|
61
|
+
onMessage(`Pulling pieces ${offset + 1}-${offset + pieces.length} of ${pullPieces.length}: ${completed} completed, ${failed} failed`);
|
|
62
|
+
},
|
|
63
|
+
});
|
|
64
|
+
results.push(...pullResult.pieces);
|
|
65
|
+
offset += pieces.length;
|
|
66
|
+
}
|
|
67
|
+
return results;
|
|
68
|
+
}
|
|
69
|
+
function createAddPiecesWorker({ localDb, targetCids, pullChunk, repair, client, state, log, }) {
|
|
13
70
|
return async (options) => {
|
|
14
71
|
let completedOps = 0;
|
|
15
72
|
let failedOps = 0;
|
|
@@ -29,18 +86,16 @@ function createAddPiecesWorker({ localDb, targetCids, repair, client, state, log
|
|
|
29
86
|
}
|
|
30
87
|
group.message(`Pulling ${operations.length} pieces...`);
|
|
31
88
|
if (operations.length > 0) {
|
|
32
|
-
const pullResult = await
|
|
89
|
+
const pullResult = await pullInChunks({
|
|
90
|
+
client,
|
|
33
91
|
serviceURL: repair.targetProviderUrl,
|
|
34
92
|
dataSetId: dataset.dataSetId,
|
|
35
93
|
clientDataSetId: dataset.clientDataSetId,
|
|
36
94
|
pieces: operationsToPullPieces(operations),
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
const failed = status.pieces.filter((piece) => piece.status === 'failed').length;
|
|
40
|
-
group.message(`Pull ${completed} completed, ${failed} failed`);
|
|
41
|
-
},
|
|
95
|
+
pullChunk,
|
|
96
|
+
onMessage: (message) => group.message(message),
|
|
42
97
|
});
|
|
43
|
-
for (const { pieceCid, status } of pullResult
|
|
98
|
+
for (const { pieceCid, status } of pullResult) {
|
|
44
99
|
const cid = pieceCid.toString();
|
|
45
100
|
if (status !== 'complete') {
|
|
46
101
|
state.failedOperations++;
|
|
@@ -148,7 +203,8 @@ export async function runAddPieces({ localDb, api, repair, concurrency, batchSiz
|
|
|
148
203
|
cursor = operations.at(-1)?.id ?? cursor;
|
|
149
204
|
return operations;
|
|
150
205
|
}
|
|
151
|
-
const
|
|
206
|
+
const pullChunk = { size: Math.max(1, Math.floor(MAX_PULL_PIECES / concurrency)) };
|
|
207
|
+
const addPiecesJob = createAddPiecesWorker({ localDb, targetCids, pullChunk, repair, client, state, log });
|
|
152
208
|
const queue = new PQueue({ concurrency });
|
|
153
209
|
while (true) {
|
|
154
210
|
await queue.onSizeLessThan(concurrency);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"add-pieces.js","sourceRoot":"","sources":["../../../src/pipeline/add-pieces.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,gBAAgB,CAAA;AACxC,OAAO,KAAK,EAAE,MAAM,wBAAwB,CAAA;AAC5C,OAAO,EAAE,GAAG,EAAE,GAAG,EAAE,EAAE,EAAE,EAAE,EAAE,OAAO,EAAE,MAAM,aAAa,CAAA;AACvD,OAAO,MAAM,MAAM,SAAS,CAAA;AAC5B,OAAO,EAAE,iBAAiB,EAAE,MAAM,WAAW,CAAA;AAC7C,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAA;AACjD,OAAO,EAAE,gBAAgB,EAAE,MAAM,6BAA6B,CAAA;AAC9D,OAAO,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAA;AACrD,OAAO,EAAE,gBAAgB,EAAE,MAAM,4BAA4B,CAAA;AAG7D,OAAO,EACL,2BAA2B,EAC3B,sBAAsB,EACtB,iBAAiB,EACjB,QAAQ,EACR,qBAAqB,EACrB,sBAAsB,GACvB,MAAM,aAAa,CAAA;AAGpB,MAAM,oBAAoB,GAAG,GAAG,CAAA;
|
|
1
|
+
{"version":3,"file":"add-pieces.js","sourceRoot":"","sources":["../../../src/pipeline/add-pieces.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,gBAAgB,CAAA;AACxC,OAAO,EAAE,SAAS,EAAE,MAAM,4BAA4B,CAAA;AACtD,OAAO,KAAK,EAAE,MAAM,wBAAwB,CAAA;AAC5C,OAAO,EAAE,aAAa,EAAE,MAAM,gCAAgC,CAAA;AAC9D,OAAO,EAAE,GAAG,EAAE,GAAG,EAAE,EAAE,EAAE,EAAE,EAAE,OAAO,EAAE,MAAM,aAAa,CAAA;AACvD,OAAO,MAAM,MAAM,SAAS,CAAA;AAC5B,OAAO,EAAE,iBAAiB,EAAE,MAAM,WAAW,CAAA;AAC7C,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAA;AACjD,OAAO,EAAE,gBAAgB,EAAE,MAAM,6BAA6B,CAAA;AAC9D,OAAO,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAA;AACrD,OAAO,EAAE,gBAAgB,EAAE,MAAM,4BAA4B,CAAA;AAG7D,OAAO,EACL,2BAA2B,EAC3B,sBAAsB,EACtB,iBAAiB,EACjB,QAAQ,EACR,qBAAqB,EACrB,sBAAsB,GACvB,MAAM,aAAa,CAAA;AAGpB,MAAM,oBAAoB,GAAG,GAAG,CAAA;AAOhC,MAAM,eAAe,GAAG,GAAG,CAAA;AAM3B,MAAM,eAAe,GAAG,EAAE,CAAA;AAG1B,MAAM,gBAAgB,GAAG,EAAE,CAAA;AAG3B,SAAS,kBAAkB,CAAC,KAAc;IACxC,OAAO,SAAS,CAAC,EAAE,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,OAAO,EAAE,QAAQ,CAAC,yBAAyB,CAAC,KAAK,IAAI,CAAA;AAC3F,CAAC;AAqCD,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,EACjC,MAAM,EACN,UAAU,EACV,SAAS,EACT,eAAe,EACf,MAAM,EAAE,UAAU,EAClB,SAAS,EACT,SAAS,GASV;IACC,MAAM,OAAO,GAAuC,EAAE,CAAA;IACtD,IAAI,MAAM,GAAG,CAAC,CAAA;IACd,OAAO,MAAM,GAAG,UAAU,CAAC,MAAM,EAAE,CAAC;QAClC,MAAM,MAAM,GAAG,UAAU,CAAC,KAAK,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC,IAAI,CAAC,CAAA;QAEhE,MAAM,SAAS,GAAG,MAAM,aAAa,CAAC,MAAM,EAAE;YAC5C,eAAe;YACf,MAAM,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC;SACrD,CAAC,CAAA;QAEF,IAAI,MAAM,CAAC,MAAM,GAAG,SAAS,CAAC,IAAI,EAAE,CAAC;YACnC,SAAQ;QACV,CAAC;QAGD,IAAI,MAAM,CAAC,MAAM,GAAG,eAAe,EAAE,CAAC;YACpC,IAAI,CAAC;gBAEH,MAAM,EAAE,CAAC,oBAAoB,CAAC;oBAC5B,UAAU;oBACV,YAAY,EAAE,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO;oBACjD,SAAS;oBACT,SAAS;oBACT,MAAM,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,EAAE,SAAS,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,QAAQ,EAAE,QAAQ,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,CAAC,CAAC;oBAC/F,UAAU,EAAE,CAAC;iBACd,CAAC,CAAA;YACJ,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,IAAI,kBAAkB,CAAC,KAAK,CAAC,EAAE,CAAC;oBAC9B,SAAS,CAAC,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,eAAe,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,EAAE,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAA;oBACnG,SAAS,CAAC,qCAAqC,SAAS,CAAC,IAAI,mBAAmB,CAAC,CAAA;oBACjF,SAAQ;gBACV,CAAC;YAEH,CAAC;QACH,CAAC;QAED,MAAM,UAAU,GAAG,MAAM,EAAE,CAAC,iBAAiB,CAAC,MAAM,EAAE;YACpD,UAAU;YACV,SAAS;YACT,eAAe;YACf,MAAM;YACN,SAAS;YACT,UAAU,EAAE,gBAAgB;YAC5B,QAAQ,EAAE,CAAC,MAAM,EAAE,EAAE;gBACnB,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,KAAK,UAAU,CAAC,CAAC,MAAM,CAAA;gBACrF,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,MAAM,CAAA;gBAChF,SAAS,CACP,kBAAkB,MAAM,GAAG,CAAC,IAAI,MAAM,GAAG,MAAM,CAAC,MAAM,OAAO,UAAU,CAAC,MAAM,KAAK,SAAS,eAAe,MAAM,SAAS,CAC3H,CAAA;YACH,CAAC;SACF,CAAC,CAAA;QACF,OAAO,CAAC,IAAI,CAAC,GAAG,UAAU,CAAC,MAAM,CAAC,CAAA;QAClC,MAAM,IAAI,MAAM,CAAC,MAAM,CAAA;IACzB,CAAC;IACD,OAAO,OAAO,CAAA;AAChB,CAAC;AAKD,SAAS,qBAAqB,CAAC,EAC7B,OAAO,EACP,UAAU,EACV,SAAS,EACT,MAAM,EACN,MAAM,EACN,KAAK,EACL,GAAG,GAC0B;IAC7B,OAAO,KAAK,EAAE,OAA4B,EAAE,EAAE;QAC5C,IAAI,YAAY,GAAG,CAAC,CAAA;QACpB,IAAI,SAAS,GAAG,CAAC,CAAA;QACjB,IAAI,UAAU,GAAsB,OAAO,CAAC,UAAU,CAAA;QAEtD,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC,SAAS,OAAO,CAAC,WAAW,KAAK,OAAO,CAAC,UAAU,CAAC,MAAM,UAAU,CAAC,CAAA;QAE7F,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,MAAM,gBAAgB,CAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,CAAC,EAAE,EAAE,MAAM,EAAE,CAAC,CAAA;YAGhF,IAAI,UAAU,EAAE,CAAC;gBACf,UAAU,GAAG,MAAM,UAAU,CAAC,EAAE,OAAO,EAAE,YAAY,EAAE,UAAU,EAAE,UAAU,EAAE,CAAC,CAAA;YAClF,CAAC;YAED,MAAM,iCAAiC,GAAG,UAAU,CAAC,MAAM,CAAA;YAC3D,UAAU,GAAG,MAAM,2BAA2B,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC,CAAA;YAC/E,MAAM,mBAAmB,GAAG,iCAAiC,GAAG,UAAU,CAAC,MAAM,CAAA;YACjF,IAAI,mBAAmB,GAAG,CAAC,EAAE,CAAC;gBAC5B,KAAK,CAAC,mBAAmB,IAAI,mBAAmB,CAAA;gBAChD,YAAY,IAAI,mBAAmB,CAAA;YACrC,CAAC;YAED,KAAK,CAAC,OAAO,CAAC,WAAW,UAAU,CAAC,MAAM,YAAY,CAAC,CAAA;YAEvD,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC1B,MAAM,UAAU,GAAG,MAAM,YAAY,CAAC;oBACpC,MAAM;oBACN,UAAU,EAAE,MAAM,CAAC,iBAAiB;oBACpC,SAAS,EAAE,OAAO,CAAC,SAAS;oBAC5B,eAAe,EAAE,OAAO,CAAC,eAAe;oBACxC,MAAM,EAAE,sBAAsB,CAAC,UAAU,CAAC;oBAC1C,SAAS;oBACT,SAAS,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC;iBAC/C,CAAC,CAAA;gBAEF,KAAK,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,UAAU,EAAE,CAAC;oBAC9C,MAAM,GAAG,GAAG,QAAQ,CAAC,QAAQ,EAAE,CAAA;oBAC/B,IAAI,MAAM,KAAK,UAAU,EAAE,CAAC;wBAC1B,KAAK,CAAC,gBAAgB,EAAE,CAAA;wBACxB,SAAS,EAAE,CAAA;wBACX,MAAM,EAAE,iBAAiB,EAAE,iBAAiB,EAAE,GAAG,sBAAsB,CAAC,UAAU,EAAE,GAAG,CAAC,CAAA;wBACxF,UAAU,GAAG,iBAAiB,CAAA;wBAC9B,MAAM,gBAAgB,CAAC;4BACrB,OAAO;4BACP,UAAU,EAAE,iBAAiB,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;gCAChD,GAAG,SAAS;gCACZ,MAAM,EAAE,QAAQ;gCAChB,KAAK,EAAE,2BAA2B,MAAM,EAAE;6BAC3C,CAAC,CAAC;yBACJ,CAAC,CAAA;oBACJ,CAAC;gBACH,CAAC;YACH,CAAC;YAGD,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC1B,KAAK,CAAC,OAAO,CAAC,UAAU,UAAU,CAAC,MAAM,YAAY,CAAC,CAAA;gBACtD,MAAM,eAAe,GAAG,MAAM,EAAE,CAAC,SAAS,CAAC,MAAM,EAAE;oBACjD,UAAU,EAAE,MAAM,CAAC,iBAAiB;oBACpC,SAAS,EAAE,OAAO,CAAC,SAAS;oBAC5B,eAAe,EAAE,OAAO,CAAC,eAAe;oBACxC,MAAM,EAAE,qBAAqB,CAAC,UAAU,EAAE,CAAC,UAAU,CAAC;iBACvD,CAAC,CAAA;gBACF,MAAM,gBAAgB,CAAC;oBACrB,OAAO;oBACP,UAAU,EAAE,UAAU,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;wBACzC,GAAG,SAAS;wBACZ,MAAM,EAAE,eAAe,CAAC,MAAM;qBAC/B,CAAC,CAAC;iBACJ,CAAC,CAAA;gBAEF,KAAK,CAAC,OAAO,CAAC,0BAA0B,QAAQ,CAAC,eAAe,CAAC,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAA;gBAC5F,MAAM,gBAAgB,GAAG,MAAM,EAAE,CAAC,gBAAgB,CAAC,eAAe,CAAC,CAAA;gBAEnE,MAAM,eAAe,GAAG,gBAAgB,CAAC,eAAe,IAAI,gBAAgB,CAAC,MAAM,CAAA;gBACnF,KAAK,CAAC,mBAAmB,IAAI,UAAU,CAAC,MAAM,CAAA;gBAC9C,YAAY,IAAI,UAAU,CAAC,MAAM,CAAA;gBACjC,KAAK,MAAM,SAAS,IAAI,UAAU;oBAAE,UAAU,EAAE,GAAG,CAAC,SAAS,CAAC,GAAG,CAAC,CAAA;gBAClE,MAAM,gBAAgB,CAAC;oBACrB,OAAO;oBACP,UAAU,EAAE,UAAU,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;wBACzC,GAAG,SAAS;wBACZ,MAAM,EAAE,WAAW;wBACnB,KAAK,EAAE,IAAI;wBACX,MAAM,EAAE,eAAe;qBACxB,CAAC,CAAC;iBACJ,CAAC,CAAA;YACJ,CAAC;YACD,KAAK,CAAC,OAAO,CAAC,SAAS,YAAY,WAAW,SAAS,SAAS,CAAC,CAAA;QACnE,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,KAAK,CAAC,gBAAgB,IAAI,UAAU,CAAC,MAAM,CAAA;YAC3C,MAAM,OAAO,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,CAAA;YACxE,KAAK,CAAC,OAAO,CAAC,GAAG,OAAO,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,EAAE,CAAC,CAAA;YAC/C,MAAM,gBAAgB,CAAC;gBACrB,OAAO;gBACP,UAAU,EAAE,UAAU,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;oBACzC,GAAG,SAAS;oBACZ,MAAM,EAAE,QAAQ;oBAChB,KAAK,EAAE,OAAO;iBACf,CAAC,CAAC;aACJ,CAAC,CAAA;QACJ,CAAC;IACH,CAAC,CAAA;AACH,CAAC;AAKD,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,EACjC,OAAO,EACP,GAAG,EACH,MAAM,EACN,WAAW,EACX,SAAS,EACT,MAAM,GACoB;IAC1B,MAAM,WAAW,GAAG,OAAO,CAAC,CAAC,CAAC,UAAU,CAAA;IACxC,IAAI,MAAM,GAAG,CAAC,CAAA;IAEd,MAAM,eAAe,GAAG,MAAM,OAAO,CAAC,MAAM,CAC1C,WAAW,CAAC,UAAU,EACtB,GAAG,CACD,EAAE,CAAC,WAAW,CAAC,UAAU,CAAC,QAAQ,EAAE,MAAM,CAAC,EAAE,CAAC,EAC9C,EAAE,CAAC,WAAW,CAAC,UAAU,CAAC,IAAI,EAAE,WAAW,CAAC,EAC5C,OAAO,CAAC,WAAW,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC,CAC9D,CACF,CAAA;IACD,IAAI,WAAW,GAAG,CAAC,CAAA;IACnB,MAAM,KAAK,GAAG;QACZ,eAAe;QACf,mBAAmB,EAAE,CAAC;QACtB,gBAAgB,EAAE,CAAC;KACpB,CAAA;IAED,MAAM,GAAG,GAAG,OAAO,CAAC;QAClB,KAAK,EAAE,eAAe;QACtB,KAAK,EAAE,CAAC;KACT,CAAC,CAAA;IAEF,MAAM,OAAO,GAAG,eAAe,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,gBAAgB,CAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,CAAC,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC,SAAS,CAAA;IAGlH,IAAI,UAAmC,CAAA;IACvC,IAAI,MAAM,CAAC,eAAe,IAAI,IAAI,IAAI,OAAO,EAAE,CAAC;QAC9C,MAAM,MAAM,GAAG,MAAM,iBAAiB,CAAC,GAAG,EAAE,OAAO,CAAC,SAAS,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAA;QAClF,UAAU,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,IAAI,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAA;IACzF,CAAC;IAED,KAAK,UAAU,YAAY;QACzB,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,OAAO,IAAI,CAAA;QACb,CAAC;QACD,MAAM,UAAU,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,UAAU,CAAC,QAAQ,CAAC;YACzD,KAAK,EAAE,GAAG,CACR,EAAE,CAAC,WAAW,CAAC,UAAU,CAAC,QAAQ,EAAE,MAAM,CAAC,EAAE,CAAC,EAC9C,EAAE,CAAC,WAAW,CAAC,UAAU,CAAC,IAAI,EAAE,WAAW,CAAC,EAC5C,OAAO,CAAC,WAAW,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC,EAC7D,EAAE,CAAC,WAAW,CAAC,UAAU,CAAC,EAAE,EAAE,MAAM,CAAC,CACtC;YACD,OAAO,EAAE,CAAC,GAAG,CAAC,WAAW,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;YACzC,KAAK,EAAE,SAAS,IAAI,oBAAoB;SACzC,CAAC,CAAA;QACF,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC5B,OAAO,IAAI,CAAA;QACb,CAAC;QAGD,MAAM,UAAU,GAAG,iBAAiB,CAAC;YACnC,UAAU,EAAE,UAAU;YACtB,SAAS,EAAE,OAAO,CAAC,SAAS;YAC5B,KAAK,EAAE,MAAM,CAAC,KAAK;YACnB,YAAY,EAAE,CAAC,UAAU;SAC1B,CAAC,CAAA;QAEF,MAAM,GAAG,UAAU,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,IAAI,MAAM,CAAA;QACxC,OAAO,UAAU,CAAA;IACnB,CAAC;IAGD,MAAM,SAAS,GAAG,EAAE,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,eAAe,GAAG,WAAW,CAAC,CAAC,EAAE,CAAA;IAClF,MAAM,YAAY,GAAG,qBAAqB,CAAC,EAAE,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,CAAA;IAC1G,MAAM,KAAK,GAAG,IAAI,MAAM,CAAC,EAAE,WAAW,EAAE,CAAC,CAAA;IAEzC,OAAO,IAAI,EAAE,CAAC;QACZ,MAAM,KAAK,CAAC,cAAc,CAAC,WAAW,CAAC,CAAA;QACvC,MAAM,UAAU,GAAG,MAAM,YAAY,EAAE,CAAA;QACvC,IAAI,CAAC,UAAU;YAAE,MAAK;QACtB,WAAW,EAAE,CAAA;QACb,MAAM,kBAAkB,GAAG,WAAW,CAAA;QACtC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,YAAY,CAAC,EAAE,UAAU,EAAE,WAAW,EAAE,kBAAkB,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;IACrG,CAAC;IAED,MAAM,KAAK,CAAC,MAAM,EAAE,CAAA;IAEpB,GAAG,CAAC,OAAO,CAAC,SAAS,KAAK,CAAC,mBAAmB,YAAY,KAAK,CAAC,gBAAgB,SAAS,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAA;IAE7G,MAAM,YAAY,CAAC;QACjB,OAAO;QACP,QAAQ,EAAE,MAAM,CAAC,EAAE;QACnB,MAAM,EAAE,KAAK,CAAC,gBAAgB,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,WAAW;KAC5D,CAAC,CAAA;AACJ,CAAC"}
|
package/package.json
CHANGED
package/readme.md
CHANGED
|
@@ -242,7 +242,7 @@ The command first ensures the target repair dataset exists for the configured wa
|
|
|
242
242
|
|
|
243
243
|
Options:
|
|
244
244
|
|
|
245
|
-
- `--concurrency <number>` controls how many
|
|
245
|
+
- `--concurrency <number>` controls how many batches run at once. Defaults to `4`. Each batch pulls its pieces in chunks so that no more than 108 pieces are pulling at once across all batches, which is Curio's per-client limit. With `4`, each pull request holds up to 27 pieces. Curio lowers that limit while other clients have pulls pending; when it refuses a pull, the CLI halves the chunk size (down to 10 pieces) for the rest of the run.
|
|
246
246
|
- `--batch-size <number>` caps the number of `add_piece` operations per batch. By default each batch holds as many pieces as fit in one `addPieces` message (about 400 without piece metadata, 80 for legacy datasets).
|
|
247
247
|
- `--payer <address>` overrides the payer address used when creating or finding the target repair dataset. Defaults to the configured wallet address.
|
|
248
248
|
|
|
@@ -306,7 +306,7 @@ The command creates a new target dataset on the target provider, copying CDN and
|
|
|
306
306
|
|
|
307
307
|
Options:
|
|
308
308
|
|
|
309
|
-
- `--concurrency <number>` controls how many
|
|
309
|
+
- `--concurrency <number>` controls how many batches run at once. Defaults to `4`. Each batch pulls its pieces in chunks so that no more than 108 pieces are pulling at once across all batches, which is Curio's per-client limit. With `4`, each pull request holds up to 27 pieces. Curio lowers that limit while other clients have pulls pending; when it refuses a pull, the CLI halves the chunk size (down to 10 pieces) for the rest of the run.
|
|
310
310
|
- `--batch-size <number>` caps the number of `add_piece` operations per batch. By default each batch holds as many pieces as fit in one `addPieces` message (about 400 without piece metadata, 80 for legacy datasets).
|
|
311
311
|
- `--payer <address>` overrides the payer address used when creating the target replication dataset. Defaults to the configured wallet address.
|
|
312
312
|
|
|
@@ -375,7 +375,7 @@ The command first ensures the target dataset exists for the configured wallet an
|
|
|
375
375
|
|
|
376
376
|
Options:
|
|
377
377
|
|
|
378
|
-
- `--concurrency <number>` controls how many
|
|
378
|
+
- `--concurrency <number>` controls how many batches run at once. Defaults to `4`. Each batch pulls its pieces in chunks so that no more than 108 pieces are pulling at once across all batches, which is Curio's per-client limit. With `4`, each pull request holds up to 27 pieces. Curio lowers that limit while other clients have pulls pending; when it refuses a pull, the CLI halves the chunk size (down to 10 pieces) for the rest of the run.
|
|
379
379
|
- `--batch-size <number>` caps the number of `add_piece` operations per batch. By default each batch holds as many pieces as fit in one `addPieces` message (about 400 without piece metadata, 80 for legacy datasets).
|
|
380
380
|
- `--payer <address>` overrides the payer address used when creating or finding the target dataset. Defaults to the configured wallet address.
|
|
381
381
|
|
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
import { taskLog } from '@clack/prompts'
|
|
2
|
+
import { PullError } from '@filoz/synapse-core/errors'
|
|
2
3
|
import * as SP from '@filoz/synapse-core/sp'
|
|
4
|
+
import { signAddPieces } from '@filoz/synapse-core/typed-data'
|
|
3
5
|
import { and, asc, eq, gt, inArray } from 'drizzle-orm'
|
|
4
6
|
import PQueue from 'p-queue'
|
|
5
7
|
import { listDataSetPieces } from '../api.ts'
|
|
@@ -21,6 +23,27 @@ import {
|
|
|
21
23
|
/** Operations read per batch before fitting; above the ~400 metadata-free pieces one addPieces message holds. */
|
|
22
24
|
const MAX_BATCH_CANDIDATES = 500
|
|
23
25
|
|
|
26
|
+
/**
|
|
27
|
+
* Curio admits at most 108 unique pieces per pull request and 108 pending pulled pieces per client.
|
|
28
|
+
*
|
|
29
|
+
* @see https://github.com/filecoin-project/curio/blob/main/pdp/pull_types.go
|
|
30
|
+
*/
|
|
31
|
+
const MAX_PULL_PIECES = 108
|
|
32
|
+
|
|
33
|
+
/**
|
|
34
|
+
* Other clients' pending pieces lower our per-client limit, but never below 10.
|
|
35
|
+
* Pulls this small are refused only when the whole node is full.
|
|
36
|
+
*/
|
|
37
|
+
const MIN_PULL_PIECES = 10
|
|
38
|
+
|
|
39
|
+
/** Curio answers 429 with Retry-After (1-5 minutes) while its pull queue is full; wait it out instead of failing. */
|
|
40
|
+
const PULL_RETRY_COUNT = 10
|
|
41
|
+
|
|
42
|
+
/** Curio's 429 body when a pull would exceed its pending-piece limits. */
|
|
43
|
+
function isPullBackpressure(error: unknown) {
|
|
44
|
+
return PullError.is(error) && error.details?.includes('pull queue backpressure') === true
|
|
45
|
+
}
|
|
46
|
+
|
|
24
47
|
export type RunPullPiecesPhaseOptions = {
|
|
25
48
|
localDb: LocalDatabase
|
|
26
49
|
api: IndexerApi
|
|
@@ -35,6 +58,8 @@ type CreateAddPiecesWorkerOptions = {
|
|
|
35
58
|
localDb: LocalDatabase
|
|
36
59
|
/** CIDs already in the target dataset; repairs only. Updated as batches are added. */
|
|
37
60
|
targetCids?: Set<string>
|
|
61
|
+
/** Max unique pieces per pull request, shared by all workers and lowered when Curio pushes back. */
|
|
62
|
+
pullChunk: { size: number }
|
|
38
63
|
repair: RepairSelect
|
|
39
64
|
client: WalletClient
|
|
40
65
|
state: {
|
|
@@ -50,10 +75,96 @@ type AddPiecesJobOptions = {
|
|
|
50
75
|
batchNumber: number
|
|
51
76
|
}
|
|
52
77
|
|
|
78
|
+
/**
|
|
79
|
+
* Pull pieces in chunks Curio admits, one chunk at a time.
|
|
80
|
+
* When Curio refuses a chunk for backpressure, halve `pullChunk.size` (shared with other workers) and try again.
|
|
81
|
+
*/
|
|
82
|
+
export async function pullInChunks({
|
|
83
|
+
client,
|
|
84
|
+
serviceURL,
|
|
85
|
+
dataSetId,
|
|
86
|
+
clientDataSetId,
|
|
87
|
+
pieces: pullPieces,
|
|
88
|
+
pullChunk,
|
|
89
|
+
onMessage,
|
|
90
|
+
}: {
|
|
91
|
+
client: WalletClient
|
|
92
|
+
serviceURL: string
|
|
93
|
+
dataSetId: bigint
|
|
94
|
+
clientDataSetId: bigint
|
|
95
|
+
pieces: SP.PullPieceInput[]
|
|
96
|
+
pullChunk: { size: number }
|
|
97
|
+
onMessage: (message: string) => void
|
|
98
|
+
}): Promise<SP.pullPieces.ReturnType['pieces']> {
|
|
99
|
+
const results: SP.pullPieces.ReturnType['pieces'] = []
|
|
100
|
+
let offset = 0
|
|
101
|
+
while (offset < pullPieces.length) {
|
|
102
|
+
const pieces = pullPieces.slice(offset, offset + pullChunk.size)
|
|
103
|
+
// Sign once so the admission check and the wait below share one pull.
|
|
104
|
+
const extraData = await signAddPieces(client, {
|
|
105
|
+
clientDataSetId,
|
|
106
|
+
pieces: pieces.map(({ pieceCid }) => ({ pieceCid })),
|
|
107
|
+
})
|
|
108
|
+
// Another worker may have shrunk the shared size while this chunk was signed; re-slice to match.
|
|
109
|
+
if (pieces.length > pullChunk.size) {
|
|
110
|
+
continue
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
// Another client's pending pull lowers our limit; shrink the chunk instead of waiting on the same size.
|
|
114
|
+
if (pieces.length > MIN_PULL_PIECES) {
|
|
115
|
+
try {
|
|
116
|
+
// Single POST: the SDK's own retries would wait out Retry-After on the same size.
|
|
117
|
+
await SP.pullPiecesApiRequest({
|
|
118
|
+
serviceURL,
|
|
119
|
+
recordKeeper: client.chain.contracts.fwss.address,
|
|
120
|
+
extraData,
|
|
121
|
+
dataSetId,
|
|
122
|
+
pieces: pieces.map(({ pieceCid, sourceUrl }) => ({ pieceCid: pieceCid.toString(), sourceUrl })),
|
|
123
|
+
retryCount: 0,
|
|
124
|
+
})
|
|
125
|
+
} catch (error) {
|
|
126
|
+
if (isPullBackpressure(error)) {
|
|
127
|
+
pullChunk.size = Math.max(MIN_PULL_PIECES, Math.floor(Math.min(pullChunk.size, pieces.length) / 2))
|
|
128
|
+
onMessage(`Curio pull queue is busy, pulling ${pullChunk.size} pieces at a time`)
|
|
129
|
+
continue
|
|
130
|
+
}
|
|
131
|
+
// Leave other errors to the retries in waitForPullPieces.
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
const pullResult = await SP.waitForPullPieces(client, {
|
|
136
|
+
serviceURL,
|
|
137
|
+
dataSetId,
|
|
138
|
+
clientDataSetId,
|
|
139
|
+
pieces,
|
|
140
|
+
extraData,
|
|
141
|
+
retryCount: PULL_RETRY_COUNT,
|
|
142
|
+
onStatus: (status) => {
|
|
143
|
+
const completed = status.pieces.filter((piece) => piece.status === 'complete').length
|
|
144
|
+
const failed = status.pieces.filter((piece) => piece.status === 'failed').length
|
|
145
|
+
onMessage(
|
|
146
|
+
`Pulling pieces ${offset + 1}-${offset + pieces.length} of ${pullPieces.length}: ${completed} completed, ${failed} failed`
|
|
147
|
+
)
|
|
148
|
+
},
|
|
149
|
+
})
|
|
150
|
+
results.push(...pullResult.pieces)
|
|
151
|
+
offset += pieces.length
|
|
152
|
+
}
|
|
153
|
+
return results
|
|
154
|
+
}
|
|
155
|
+
|
|
53
156
|
/**
|
|
54
157
|
* Create a worker function that adds pieces to the target repair dataset.
|
|
55
158
|
*/
|
|
56
|
-
function createAddPiecesWorker({
|
|
159
|
+
function createAddPiecesWorker({
|
|
160
|
+
localDb,
|
|
161
|
+
targetCids,
|
|
162
|
+
pullChunk,
|
|
163
|
+
repair,
|
|
164
|
+
client,
|
|
165
|
+
state,
|
|
166
|
+
log,
|
|
167
|
+
}: CreateAddPiecesWorkerOptions) {
|
|
57
168
|
return async (options: AddPiecesJobOptions) => {
|
|
58
169
|
let completedOps = 0
|
|
59
170
|
let failedOps = 0
|
|
@@ -78,21 +189,19 @@ function createAddPiecesWorker({ localDb, targetCids, repair, client, state, log
|
|
|
78
189
|
}
|
|
79
190
|
|
|
80
191
|
group.message(`Pulling ${operations.length} pieces...`)
|
|
81
|
-
// pull pieces
|
|
192
|
+
// pull pieces in chunks that Curio admits, one chunk at a time per worker
|
|
82
193
|
if (operations.length > 0) {
|
|
83
|
-
const pullResult = await
|
|
194
|
+
const pullResult = await pullInChunks({
|
|
195
|
+
client,
|
|
84
196
|
serviceURL: repair.targetProviderUrl,
|
|
85
197
|
dataSetId: dataset.dataSetId,
|
|
86
198
|
clientDataSetId: dataset.clientDataSetId,
|
|
87
199
|
pieces: operationsToPullPieces(operations),
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
const failed = status.pieces.filter((piece) => piece.status === 'failed').length
|
|
91
|
-
group.message(`Pull ${completed} completed, ${failed} failed`)
|
|
92
|
-
},
|
|
200
|
+
pullChunk,
|
|
201
|
+
onMessage: (message) => group.message(message),
|
|
93
202
|
})
|
|
94
203
|
|
|
95
|
-
for (const { pieceCid, status } of pullResult
|
|
204
|
+
for (const { pieceCid, status } of pullResult) {
|
|
96
205
|
const cid = pieceCid.toString()
|
|
97
206
|
if (status !== 'complete') {
|
|
98
207
|
state.failedOperations++
|
|
@@ -235,7 +344,9 @@ export async function runAddPieces({
|
|
|
235
344
|
return operations
|
|
236
345
|
}
|
|
237
346
|
|
|
238
|
-
|
|
347
|
+
// Keep pieces pulling across all workers within Curio's per-client limit.
|
|
348
|
+
const pullChunk = { size: Math.max(1, Math.floor(MAX_PULL_PIECES / concurrency)) }
|
|
349
|
+
const addPiecesJob = createAddPiecesWorker({ localDb, targetCids, pullChunk, repair, client, state, log })
|
|
239
350
|
const queue = new PQueue({ concurrency })
|
|
240
351
|
|
|
241
352
|
while (true) {
|