@filoz/repair-cli 1.1.0 → 1.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/package.json
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"add-pieces.d.ts","sourceRoot":"","sources":["../../../src/pipeline/add-pieces.ts"],"names":[],"mappings":"AASA,OAAO,KAAK,EAAmB,YAAY,EAAE,MAAM,oBAAoB,CAAA;AACvE,OAAO,KAAK,EAAE,UAAU,EAAE,aAAa,EAAE,YAAY,EAAE,MAAM,aAAa,CAAA;
|
|
1
|
+
{"version":3,"file":"add-pieces.d.ts","sourceRoot":"","sources":["../../../src/pipeline/add-pieces.ts"],"names":[],"mappings":"AASA,OAAO,KAAK,EAAmB,YAAY,EAAE,MAAM,oBAAoB,CAAA;AACvE,OAAO,KAAK,EAAE,UAAU,EAAE,aAAa,EAAE,YAAY,EAAE,MAAM,aAAa,CAAA;AAuB1E,MAAM,MAAM,yBAAyB,GAAG;IACtC,OAAO,EAAE,aAAa,CAAA;IACtB,GAAG,EAAE,UAAU,CAAA;IACf,MAAM,EAAE,YAAY,CAAA;IACpB,WAAW,EAAE,MAAM,CAAA;IAEnB,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,MAAM,EAAE,YAAY,CAAA;CACrB,CAAA;AAwJD,wBAAsB,YAAY,CAAC,EACjC,OAAO,EACP,GAAG,EACH,MAAM,EACN,WAAW,EACX,SAAS,EACT,MAAM,GACP,EAAE,yBAAyB,GAAG,OAAO,CAAC,IAAI,CAAC,CAsF3C"}
|
|
@@ -9,7 +9,9 @@ import { repairUpdate } from "../db/repair-update.js";
|
|
|
9
9
|
import { upsertOperations } from "../db/upsert-operations.js";
|
|
10
10
|
import { completeConfirmedOperations, excludeOperationsByCid, fitAddPiecesBatch, hashLink, operationsToAddPieces, operationsToPullPieces, } from "../utils.js";
|
|
11
11
|
const MAX_BATCH_CANDIDATES = 500;
|
|
12
|
-
|
|
12
|
+
const MAX_PULL_PIECES = 108;
|
|
13
|
+
const PULL_RETRY_COUNT = 10;
|
|
14
|
+
function createAddPiecesWorker({ localDb, targetCids, pullChunkSize, repair, client, state, log, }) {
|
|
13
15
|
return async (options) => {
|
|
14
16
|
let completedOps = 0;
|
|
15
17
|
let failedOps = 0;
|
|
@@ -29,32 +31,38 @@ function createAddPiecesWorker({ localDb, targetCids, repair, client, state, log
|
|
|
29
31
|
}
|
|
30
32
|
group.message(`Pulling ${operations.length} pieces...`);
|
|
31
33
|
if (operations.length > 0) {
|
|
32
|
-
const
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
34
|
+
const pullPieces = operationsToPullPieces(operations);
|
|
35
|
+
const pullChunks = Math.ceil(pullPieces.length / pullChunkSize);
|
|
36
|
+
for (let offset = 0; offset < pullPieces.length; offset += pullChunkSize) {
|
|
37
|
+
const chunk = offset / pullChunkSize + 1;
|
|
38
|
+
const pullResult = await SP.waitForPullPieces(client, {
|
|
39
|
+
serviceURL: repair.targetProviderUrl,
|
|
40
|
+
dataSetId: dataset.dataSetId,
|
|
41
|
+
clientDataSetId: dataset.clientDataSetId,
|
|
42
|
+
pieces: pullPieces.slice(offset, offset + pullChunkSize),
|
|
43
|
+
retryCount: PULL_RETRY_COUNT,
|
|
44
|
+
onStatus: (status) => {
|
|
45
|
+
const completed = status.pieces.filter((piece) => piece.status === 'complete').length;
|
|
46
|
+
const failed = status.pieces.filter((piece) => piece.status === 'failed').length;
|
|
47
|
+
group.message(`Pull ${chunk}/${pullChunks}: ${completed} completed, ${failed} failed`);
|
|
48
|
+
},
|
|
49
|
+
});
|
|
50
|
+
for (const { pieceCid, status } of pullResult.pieces) {
|
|
51
|
+
const cid = pieceCid.toString();
|
|
52
|
+
if (status !== 'complete') {
|
|
53
|
+
state.failedOperations++;
|
|
54
|
+
failedOps++;
|
|
55
|
+
const { operationToFailed, operationToCommit } = excludeOperationsByCid(operations, cid);
|
|
56
|
+
operations = operationToCommit;
|
|
57
|
+
await upsertOperations({
|
|
58
|
+
localDb,
|
|
59
|
+
operations: operationToFailed.map((operation) => ({
|
|
60
|
+
...operation,
|
|
61
|
+
status: 'failed',
|
|
62
|
+
error: `pull failed with status ${status}`,
|
|
63
|
+
})),
|
|
64
|
+
});
|
|
65
|
+
}
|
|
58
66
|
}
|
|
59
67
|
}
|
|
60
68
|
}
|
|
@@ -148,7 +156,8 @@ export async function runAddPieces({ localDb, api, repair, concurrency, batchSiz
|
|
|
148
156
|
cursor = operations.at(-1)?.id ?? cursor;
|
|
149
157
|
return operations;
|
|
150
158
|
}
|
|
151
|
-
const
|
|
159
|
+
const pullChunkSize = Math.max(1, Math.floor(MAX_PULL_PIECES / concurrency));
|
|
160
|
+
const addPiecesJob = createAddPiecesWorker({ localDb, targetCids, pullChunkSize, repair, client, state, log });
|
|
152
161
|
const queue = new PQueue({ concurrency });
|
|
153
162
|
while (true) {
|
|
154
163
|
await queue.onSizeLessThan(concurrency);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"add-pieces.js","sourceRoot":"","sources":["../../../src/pipeline/add-pieces.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,gBAAgB,CAAA;AACxC,OAAO,KAAK,EAAE,MAAM,wBAAwB,CAAA;AAC5C,OAAO,EAAE,GAAG,EAAE,GAAG,EAAE,EAAE,EAAE,EAAE,EAAE,OAAO,EAAE,MAAM,aAAa,CAAA;AACvD,OAAO,MAAM,MAAM,SAAS,CAAA;AAC5B,OAAO,EAAE,iBAAiB,EAAE,MAAM,WAAW,CAAA;AAC7C,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAA;AACjD,OAAO,EAAE,gBAAgB,EAAE,MAAM,6BAA6B,CAAA;AAC9D,OAAO,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAA;AACrD,OAAO,EAAE,gBAAgB,EAAE,MAAM,4BAA4B,CAAA;AAG7D,OAAO,EACL,2BAA2B,EAC3B,sBAAsB,EACtB,iBAAiB,EACjB,QAAQ,EACR,qBAAqB,EACrB,sBAAsB,GACvB,MAAM,aAAa,CAAA;AAGpB,MAAM,oBAAoB,GAAG,GAAG,CAAA;
|
|
1
|
+
{"version":3,"file":"add-pieces.js","sourceRoot":"","sources":["../../../src/pipeline/add-pieces.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,gBAAgB,CAAA;AACxC,OAAO,KAAK,EAAE,MAAM,wBAAwB,CAAA;AAC5C,OAAO,EAAE,GAAG,EAAE,GAAG,EAAE,EAAE,EAAE,EAAE,EAAE,OAAO,EAAE,MAAM,aAAa,CAAA;AACvD,OAAO,MAAM,MAAM,SAAS,CAAA;AAC5B,OAAO,EAAE,iBAAiB,EAAE,MAAM,WAAW,CAAA;AAC7C,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAA;AACjD,OAAO,EAAE,gBAAgB,EAAE,MAAM,6BAA6B,CAAA;AAC9D,OAAO,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAA;AACrD,OAAO,EAAE,gBAAgB,EAAE,MAAM,4BAA4B,CAAA;AAG7D,OAAO,EACL,2BAA2B,EAC3B,sBAAsB,EACtB,iBAAiB,EACjB,QAAQ,EACR,qBAAqB,EACrB,sBAAsB,GACvB,MAAM,aAAa,CAAA;AAGpB,MAAM,oBAAoB,GAAG,GAAG,CAAA;AAOhC,MAAM,eAAe,GAAG,GAAG,CAAA;AAG3B,MAAM,gBAAgB,GAAG,EAAE,CAAA;AAoC3B,SAAS,qBAAqB,CAAC,EAC7B,OAAO,EACP,UAAU,EACV,aAAa,EACb,MAAM,EACN,MAAM,EACN,KAAK,EACL,GAAG,GAC0B;IAC7B,OAAO,KAAK,EAAE,OAA4B,EAAE,EAAE;QAC5C,IAAI,YAAY,GAAG,CAAC,CAAA;QACpB,IAAI,SAAS,GAAG,CAAC,CAAA;QACjB,IAAI,UAAU,GAAsB,OAAO,CAAC,UAAU,CAAA;QAEtD,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC,SAAS,OAAO,CAAC,WAAW,KAAK,OAAO,CAAC,UAAU,CAAC,MAAM,UAAU,CAAC,CAAA;QAE7F,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,MAAM,gBAAgB,CAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,CAAC,EAAE,EAAE,MAAM,EAAE,CAAC,CAAA;YAGhF,IAAI,UAAU,EAAE,CAAC;gBACf,UAAU,GAAG,MAAM,UAAU,CAAC,EAAE,OAAO,EAAE,YAAY,EAAE,UAAU,EAAE,UAAU,EAAE,CAAC,CAAA;YAClF,CAAC;YAED,MAAM,iCAAiC,GAAG,UAAU,CAAC,MAAM,CAAA;YAC3D,UAAU,GAAG,MAAM,2BAA2B,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC,CAAA;YAC/E,MAAM,mBAAmB,GAAG,iCAAiC,GAAG,UAAU,CAAC,MAAM,CAAA;YACjF,IAAI,mBAAmB,GAAG,CAAC,EAAE,CAAC;gBAC5B,KAAK,CAAC,mBAAmB,IAAI,mBAAmB,CAAA;gBAChD,YAAY,IAAI,mBAAmB,CAAA;YACrC,CAAC;YAED,KAAK,CAAC,OAAO,CAAC,WAAW,UAAU,CAAC,MAAM,YAAY,CAAC,CAAA;YAEvD,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC1B,MAAM,UAAU,GAAG,sBAAsB,CAAC,UAAU,CAAC,CAAA;gBACrD,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,MAAM,GAAG,aAAa,CAAC,CAAA;gBAC/D,KAAK,IAAI,MAAM,GAAG,CAAC,EAAE,MAAM,GAAG,UAAU,CAAC,MAAM,EAAE,MAAM,IAAI,aAAa,EAAE,CAAC;oBACzE,MAAM,KAAK,GAAG,MAAM,GAAG,aAAa,GAAG,CAAC,CAAA;oBACxC,MAAM,UAAU,GAAG,MAAM,EAAE,CAAC,iBAAiB,CAAC,MAAM,EAAE;wBACpD,UAAU,EAAE,MAAM,CAAC,iBAAiB;wBACpC,SAAS,EAAE,OAAO,CAAC,SAAS;wBAC5B,eAAe,EAAE,OAAO,CAAC,eAAe;wBACxC,MAAM,EAAE,UAAU,CAAC,KAAK,CAAC,MAAM,EAAE,MAAM,GAAG,aAAa,CAAC;wBACxD,UAAU,EAAE,gBAAgB;wBAC5B,QAAQ,EAAE,CAAC,MAAM,EAAE,EAAE;4BACnB,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,KAAK,UAAU,CAAC,CAAC,MAAM,CAAA;4BACrF,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,MAAM,CAAA;4BAChF,KAAK,CAAC,OAAO,CAAC,QAAQ,KAAK,IAAI,UAAU,KAAK,SAAS,eAAe,MAAM,SAAS,CAAC,CAAA;wBACxF,CAAC;qBACF,CAAC,CAAA;oBAEF,KAAK,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,UAAU,CAAC,MAAM,EAAE,CAAC;wBACrD,MAAM,GAAG,GAAG,QAAQ,CAAC,QAAQ,EAAE,CAAA;wBAC/B,IAAI,MAAM,KAAK,UAAU,EAAE,CAAC;4BAC1B,KAAK,CAAC,gBAAgB,EAAE,CAAA;4BACxB,SAAS,EAAE,CAAA;4BACX,MAAM,EAAE,iBAAiB,EAAE,iBAAiB,EAAE,GAAG,sBAAsB,CAAC,UAAU,EAAE,GAAG,CAAC,CAAA;4BACxF,UAAU,GAAG,iBAAiB,CAAA;4BAC9B,MAAM,gBAAgB,CAAC;gCACrB,OAAO;gCACP,UAAU,EAAE,iBAAiB,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;oCAChD,GAAG,SAAS;oCACZ,MAAM,EAAE,QAAQ;oCAChB,KAAK,EAAE,2BAA2B,MAAM,EAAE;iCAC3C,CAAC,CAAC;6BACJ,CAAC,CAAA;wBACJ,CAAC;oBACH,CAAC;gBACH,CAAC;YACH,CAAC;YAGD,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC1B,KAAK,CAAC,OAAO,CAAC,UAAU,UAAU,CAAC,MAAM,YAAY,CAAC,CAAA;gBACtD,MAAM,eAAe,GAAG,MAAM,EAAE,CAAC,SAAS,CAAC,MAAM,EAAE;oBACjD,UAAU,EAAE,MAAM,CAAC,iBAAiB;oBACpC,SAAS,EAAE,OAAO,CAAC,SAAS;oBAC5B,eAAe,EAAE,OAAO,CAAC,eAAe;oBACxC,MAAM,EAAE,qBAAqB,CAAC,UAAU,EAAE,CAAC,UAAU,CAAC;iBACvD,CAAC,CAAA;gBACF,MAAM,gBAAgB,CAAC;oBACrB,OAAO;oBACP,UAAU,EAAE,UAAU,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;wBACzC,GAAG,SAAS;wBACZ,MAAM,EAAE,eAAe,CAAC,MAAM;qBAC/B,CAAC,CAAC;iBACJ,CAAC,CAAA;gBAEF,KAAK,CAAC,OAAO,CAAC,0BAA0B,QAAQ,CAAC,eAAe,CAAC,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAA;gBAC5F,MAAM,gBAAgB,GAAG,MAAM,EAAE,CAAC,gBAAgB,CAAC,eAAe,CAAC,CAAA;gBAEnE,MAAM,eAAe,GAAG,gBAAgB,CAAC,eAAe,IAAI,gBAAgB,CAAC,MAAM,CAAA;gBACnF,KAAK,CAAC,mBAAmB,IAAI,UAAU,CAAC,MAAM,CAAA;gBAC9C,YAAY,IAAI,UAAU,CAAC,MAAM,CAAA;gBACjC,KAAK,MAAM,SAAS,IAAI,UAAU;oBAAE,UAAU,EAAE,GAAG,CAAC,SAAS,CAAC,GAAG,CAAC,CAAA;gBAClE,MAAM,gBAAgB,CAAC;oBACrB,OAAO;oBACP,UAAU,EAAE,UAAU,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;wBACzC,GAAG,SAAS;wBACZ,MAAM,EAAE,WAAW;wBACnB,KAAK,EAAE,IAAI;wBACX,MAAM,EAAE,eAAe;qBACxB,CAAC,CAAC;iBACJ,CAAC,CAAA;YACJ,CAAC;YACD,KAAK,CAAC,OAAO,CAAC,SAAS,YAAY,WAAW,SAAS,SAAS,CAAC,CAAA;QACnE,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,KAAK,CAAC,gBAAgB,IAAI,UAAU,CAAC,MAAM,CAAA;YAC3C,MAAM,OAAO,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,CAAA;YACxE,KAAK,CAAC,OAAO,CAAC,GAAG,OAAO,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,EAAE,CAAC,CAAA;YAC/C,MAAM,gBAAgB,CAAC;gBACrB,OAAO;gBACP,UAAU,EAAE,UAAU,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;oBACzC,GAAG,SAAS;oBACZ,MAAM,EAAE,QAAQ;oBAChB,KAAK,EAAE,OAAO;iBACf,CAAC,CAAC;aACJ,CAAC,CAAA;QACJ,CAAC;IACH,CAAC,CAAA;AACH,CAAC;AAKD,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,EACjC,OAAO,EACP,GAAG,EACH,MAAM,EACN,WAAW,EACX,SAAS,EACT,MAAM,GACoB;IAC1B,MAAM,WAAW,GAAG,OAAO,CAAC,CAAC,CAAC,UAAU,CAAA;IACxC,IAAI,MAAM,GAAG,CAAC,CAAA;IAEd,MAAM,eAAe,GAAG,MAAM,OAAO,CAAC,MAAM,CAC1C,WAAW,CAAC,UAAU,EACtB,GAAG,CACD,EAAE,CAAC,WAAW,CAAC,UAAU,CAAC,QAAQ,EAAE,MAAM,CAAC,EAAE,CAAC,EAC9C,EAAE,CAAC,WAAW,CAAC,UAAU,CAAC,IAAI,EAAE,WAAW,CAAC,EAC5C,OAAO,CAAC,WAAW,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC,CAC9D,CACF,CAAA;IACD,IAAI,WAAW,GAAG,CAAC,CAAA;IACnB,MAAM,KAAK,GAAG;QACZ,eAAe;QACf,mBAAmB,EAAE,CAAC;QACtB,gBAAgB,EAAE,CAAC;KACpB,CAAA;IAED,MAAM,GAAG,GAAG,OAAO,CAAC;QAClB,KAAK,EAAE,eAAe;QACtB,KAAK,EAAE,CAAC;KACT,CAAC,CAAA;IAEF,MAAM,OAAO,GAAG,eAAe,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,gBAAgB,CAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,CAAC,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC,SAAS,CAAA;IAGlH,IAAI,UAAmC,CAAA;IACvC,IAAI,MAAM,CAAC,eAAe,IAAI,IAAI,IAAI,OAAO,EAAE,CAAC;QAC9C,MAAM,MAAM,GAAG,MAAM,iBAAiB,CAAC,GAAG,EAAE,OAAO,CAAC,SAAS,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAA;QAClF,UAAU,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,IAAI,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAA;IACzF,CAAC;IAED,KAAK,UAAU,YAAY;QACzB,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,OAAO,IAAI,CAAA;QACb,CAAC;QACD,MAAM,UAAU,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,UAAU,CAAC,QAAQ,CAAC;YACzD,KAAK,EAAE,GAAG,CACR,EAAE,CAAC,WAAW,CAAC,UAAU,CAAC,QAAQ,EAAE,MAAM,CAAC,EAAE,CAAC,EAC9C,EAAE,CAAC,WAAW,CAAC,UAAU,CAAC,IAAI,EAAE,WAAW,CAAC,EAC5C,OAAO,CAAC,WAAW,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC,EAC7D,EAAE,CAAC,WAAW,CAAC,UAAU,CAAC,EAAE,EAAE,MAAM,CAAC,CACtC;YACD,OAAO,EAAE,CAAC,GAAG,CAAC,WAAW,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;YACzC,KAAK,EAAE,SAAS,IAAI,oBAAoB;SACzC,CAAC,CAAA;QACF,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC5B,OAAO,IAAI,CAAA;QACb,CAAC;QAGD,MAAM,UAAU,GAAG,iBAAiB,CAAC;YACnC,UAAU,EAAE,UAAU;YACtB,SAAS,EAAE,OAAO,CAAC,SAAS;YAC5B,KAAK,EAAE,MAAM,CAAC,KAAK;YACnB,YAAY,EAAE,CAAC,UAAU;SAC1B,CAAC,CAAA;QAEF,MAAM,GAAG,UAAU,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,IAAI,MAAM,CAAA;QACxC,OAAO,UAAU,CAAA;IACnB,CAAC;IAGD,MAAM,aAAa,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,eAAe,GAAG,WAAW,CAAC,CAAC,CAAA;IAC5E,MAAM,YAAY,GAAG,qBAAqB,CAAC,EAAE,OAAO,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,CAAA;IAC9G,MAAM,KAAK,GAAG,IAAI,MAAM,CAAC,EAAE,WAAW,EAAE,CAAC,CAAA;IAEzC,OAAO,IAAI,EAAE,CAAC;QACZ,MAAM,KAAK,CAAC,cAAc,CAAC,WAAW,CAAC,CAAA;QACvC,MAAM,UAAU,GAAG,MAAM,YAAY,EAAE,CAAA;QACvC,IAAI,CAAC,UAAU;YAAE,MAAK;QACtB,WAAW,EAAE,CAAA;QACb,MAAM,kBAAkB,GAAG,WAAW,CAAA;QACtC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,YAAY,CAAC,EAAE,UAAU,EAAE,WAAW,EAAE,kBAAkB,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;IACrG,CAAC;IAED,MAAM,KAAK,CAAC,MAAM,EAAE,CAAA;IAEpB,GAAG,CAAC,OAAO,CAAC,SAAS,KAAK,CAAC,mBAAmB,YAAY,KAAK,CAAC,gBAAgB,SAAS,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAA;IAE7G,MAAM,YAAY,CAAC;QACjB,OAAO;QACP,QAAQ,EAAE,MAAM,CAAC,EAAE;QACnB,MAAM,EAAE,KAAK,CAAC,gBAAgB,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,WAAW;KAC5D,CAAC,CAAA;AACJ,CAAC"}
|
package/package.json
CHANGED
package/readme.md
CHANGED
|
@@ -242,7 +242,7 @@ The command first ensures the target repair dataset exists for the configured wa
|
|
|
242
242
|
|
|
243
243
|
Options:
|
|
244
244
|
|
|
245
|
-
- `--concurrency <number>` controls how many
|
|
245
|
+
- `--concurrency <number>` controls how many batches run at once. Defaults to `4`. Each batch pulls its pieces in chunks so that no more than 108 pieces are pulling at once across all batches, which is Curio's per-client limit. With `4`, each pull request holds up to 27 pieces.
|
|
246
246
|
- `--batch-size <number>` caps the number of `add_piece` operations per batch. By default each batch holds as many pieces as fit in one `addPieces` message (about 400 without piece metadata, 80 for legacy datasets).
|
|
247
247
|
- `--payer <address>` overrides the payer address used when creating or finding the target repair dataset. Defaults to the configured wallet address.
|
|
248
248
|
|
|
@@ -306,7 +306,7 @@ The command creates a new target dataset on the target provider, copying CDN and
|
|
|
306
306
|
|
|
307
307
|
Options:
|
|
308
308
|
|
|
309
|
-
- `--concurrency <number>` controls how many
|
|
309
|
+
- `--concurrency <number>` controls how many batches run at once. Defaults to `4`. Each batch pulls its pieces in chunks so that no more than 108 pieces are pulling at once across all batches, which is Curio's per-client limit. With `4`, each pull request holds up to 27 pieces.
|
|
310
310
|
- `--batch-size <number>` caps the number of `add_piece` operations per batch. By default each batch holds as many pieces as fit in one `addPieces` message (about 400 without piece metadata, 80 for legacy datasets).
|
|
311
311
|
- `--payer <address>` overrides the payer address used when creating the target replication dataset. Defaults to the configured wallet address.
|
|
312
312
|
|
|
@@ -375,7 +375,7 @@ The command first ensures the target dataset exists for the configured wallet an
|
|
|
375
375
|
|
|
376
376
|
Options:
|
|
377
377
|
|
|
378
|
-
- `--concurrency <number>` controls how many
|
|
378
|
+
- `--concurrency <number>` controls how many batches run at once. Defaults to `4`. Each batch pulls its pieces in chunks so that no more than 108 pieces are pulling at once across all batches, which is Curio's per-client limit. With `4`, each pull request holds up to 27 pieces.
|
|
379
379
|
- `--batch-size <number>` caps the number of `add_piece` operations per batch. By default each batch holds as many pieces as fit in one `addPieces` message (about 400 without piece metadata, 80 for legacy datasets).
|
|
380
380
|
- `--payer <address>` overrides the payer address used when creating or finding the target dataset. Defaults to the configured wallet address.
|
|
381
381
|
|
|
@@ -21,6 +21,16 @@ import {
|
|
|
21
21
|
/** Operations read per batch before fitting; above the ~400 metadata-free pieces one addPieces message holds. */
|
|
22
22
|
const MAX_BATCH_CANDIDATES = 500
|
|
23
23
|
|
|
24
|
+
/**
|
|
25
|
+
* Curio admits at most 108 unique pieces per pull request and 108 pending pulled pieces per client.
|
|
26
|
+
*
|
|
27
|
+
* @see https://github.com/filecoin-project/curio/blob/main/pdp/pull_types.go
|
|
28
|
+
*/
|
|
29
|
+
const MAX_PULL_PIECES = 108
|
|
30
|
+
|
|
31
|
+
/** Curio answers 429 with Retry-After (1-5 minutes) while its pull queue is full; wait it out instead of failing. */
|
|
32
|
+
const PULL_RETRY_COUNT = 10
|
|
33
|
+
|
|
24
34
|
export type RunPullPiecesPhaseOptions = {
|
|
25
35
|
localDb: LocalDatabase
|
|
26
36
|
api: IndexerApi
|
|
@@ -35,6 +45,8 @@ type CreateAddPiecesWorkerOptions = {
|
|
|
35
45
|
localDb: LocalDatabase
|
|
36
46
|
/** CIDs already in the target dataset; repairs only. Updated as batches are added. */
|
|
37
47
|
targetCids?: Set<string>
|
|
48
|
+
/** Max unique pieces per pull request. */
|
|
49
|
+
pullChunkSize: number
|
|
38
50
|
repair: RepairSelect
|
|
39
51
|
client: WalletClient
|
|
40
52
|
state: {
|
|
@@ -53,7 +65,15 @@ type AddPiecesJobOptions = {
|
|
|
53
65
|
/**
|
|
54
66
|
* Create a worker function that adds pieces to the target repair dataset.
|
|
55
67
|
*/
|
|
56
|
-
function createAddPiecesWorker({
|
|
68
|
+
function createAddPiecesWorker({
|
|
69
|
+
localDb,
|
|
70
|
+
targetCids,
|
|
71
|
+
pullChunkSize,
|
|
72
|
+
repair,
|
|
73
|
+
client,
|
|
74
|
+
state,
|
|
75
|
+
log,
|
|
76
|
+
}: CreateAddPiecesWorkerOptions) {
|
|
57
77
|
return async (options: AddPiecesJobOptions) => {
|
|
58
78
|
let completedOps = 0
|
|
59
79
|
let failedOps = 0
|
|
@@ -78,35 +98,41 @@ function createAddPiecesWorker({ localDb, targetCids, repair, client, state, log
|
|
|
78
98
|
}
|
|
79
99
|
|
|
80
100
|
group.message(`Pulling ${operations.length} pieces...`)
|
|
81
|
-
// pull pieces
|
|
101
|
+
// pull pieces in chunks that Curio admits, one chunk at a time per worker
|
|
82
102
|
if (operations.length > 0) {
|
|
83
|
-
const
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
103
|
+
const pullPieces = operationsToPullPieces(operations)
|
|
104
|
+
const pullChunks = Math.ceil(pullPieces.length / pullChunkSize)
|
|
105
|
+
for (let offset = 0; offset < pullPieces.length; offset += pullChunkSize) {
|
|
106
|
+
const chunk = offset / pullChunkSize + 1
|
|
107
|
+
const pullResult = await SP.waitForPullPieces(client, {
|
|
108
|
+
serviceURL: repair.targetProviderUrl,
|
|
109
|
+
dataSetId: dataset.dataSetId,
|
|
110
|
+
clientDataSetId: dataset.clientDataSetId,
|
|
111
|
+
pieces: pullPieces.slice(offset, offset + pullChunkSize),
|
|
112
|
+
retryCount: PULL_RETRY_COUNT,
|
|
113
|
+
onStatus: (status) => {
|
|
114
|
+
const completed = status.pieces.filter((piece) => piece.status === 'complete').length
|
|
115
|
+
const failed = status.pieces.filter((piece) => piece.status === 'failed').length
|
|
116
|
+
group.message(`Pull ${chunk}/${pullChunks}: ${completed} completed, ${failed} failed`)
|
|
117
|
+
},
|
|
118
|
+
})
|
|
94
119
|
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
120
|
+
for (const { pieceCid, status } of pullResult.pieces) {
|
|
121
|
+
const cid = pieceCid.toString()
|
|
122
|
+
if (status !== 'complete') {
|
|
123
|
+
state.failedOperations++
|
|
124
|
+
failedOps++
|
|
125
|
+
const { operationToFailed, operationToCommit } = excludeOperationsByCid(operations, cid)
|
|
126
|
+
operations = operationToCommit
|
|
127
|
+
await upsertOperations({
|
|
128
|
+
localDb,
|
|
129
|
+
operations: operationToFailed.map((operation) => ({
|
|
130
|
+
...operation,
|
|
131
|
+
status: 'failed',
|
|
132
|
+
error: `pull failed with status ${status}`,
|
|
133
|
+
})),
|
|
134
|
+
})
|
|
135
|
+
}
|
|
110
136
|
}
|
|
111
137
|
}
|
|
112
138
|
}
|
|
@@ -235,7 +261,9 @@ export async function runAddPieces({
|
|
|
235
261
|
return operations
|
|
236
262
|
}
|
|
237
263
|
|
|
238
|
-
|
|
264
|
+
// Keep pieces pulling across all workers within Curio's per-client limit.
|
|
265
|
+
const pullChunkSize = Math.max(1, Math.floor(MAX_PULL_PIECES / concurrency))
|
|
266
|
+
const addPiecesJob = createAddPiecesWorker({ localDb, targetCids, pullChunkSize, repair, client, state, log })
|
|
239
267
|
const queue = new PQueue({ concurrency })
|
|
240
268
|
|
|
241
269
|
while (true) {
|