@filoz/repair-cli 1.1.1 → 1.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/package.json +1 -1
- package/dist/src/pipeline/add-pieces.d.ts +12 -0
- package/dist/src/pipeline/add-pieces.d.ts.map +1 -1
- package/dist/src/pipeline/add-pieces.js +82 -35
- package/dist/src/pipeline/add-pieces.js.map +1 -1
- package/package.json +1 -1
- package/readme.md +3 -3
- package/src/pipeline/add-pieces.ts +121 -38
package/dist/package.json
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import * as SP from '@filoz/synapse-core/sp';
|
|
1
2
|
import type { RepairSelect } from '../local-schema.ts';
|
|
2
3
|
import type { IndexerApi, LocalDatabase, WalletClient } from '../types.ts';
|
|
3
4
|
export type RunPullPiecesPhaseOptions = {
|
|
@@ -8,5 +9,16 @@ export type RunPullPiecesPhaseOptions = {
|
|
|
8
9
|
batchSize?: number;
|
|
9
10
|
client: WalletClient;
|
|
10
11
|
};
|
|
12
|
+
export declare function pullInChunks({ client, serviceURL, dataSetId, clientDataSetId, pieces: pullPieces, pullChunk, onMessage, }: {
|
|
13
|
+
client: WalletClient;
|
|
14
|
+
serviceURL: string;
|
|
15
|
+
dataSetId: bigint;
|
|
16
|
+
clientDataSetId: bigint;
|
|
17
|
+
pieces: SP.PullPieceInput[];
|
|
18
|
+
pullChunk: {
|
|
19
|
+
size: number;
|
|
20
|
+
};
|
|
21
|
+
onMessage: (message: string) => void;
|
|
22
|
+
}): Promise<SP.pullPieces.ReturnType['pieces']>;
|
|
11
23
|
export declare function runAddPieces({ localDb, api, repair, concurrency, batchSize, client, }: RunPullPiecesPhaseOptions): Promise<void>;
|
|
12
24
|
//# sourceMappingURL=add-pieces.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"add-pieces.d.ts","sourceRoot":"","sources":["../../../src/pipeline/add-pieces.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"add-pieces.d.ts","sourceRoot":"","sources":["../../../src/pipeline/add-pieces.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,MAAM,wBAAwB,CAAA;AAS5C,OAAO,KAAK,EAAmB,YAAY,EAAE,MAAM,oBAAoB,CAAA;AACvE,OAAO,KAAK,EAAE,UAAU,EAAE,aAAa,EAAE,YAAY,EAAE,MAAM,aAAa,CAAA;AAkC1E,MAAM,MAAM,yBAAyB,GAAG;IACtC,OAAO,EAAE,aAAa,CAAA;IACtB,GAAG,EAAE,UAAU,CAAA;IACf,MAAM,EAAE,YAAY,CAAA;IACpB,WAAW,EAAE,MAAM,CAAA;IAEnB,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,MAAM,EAAE,YAAY,CAAA;CACrB,CAAA;AA2BD,wBAAsB,YAAY,CAAC,EACjC,MAAM,EACN,UAAU,EACV,SAAS,EACT,eAAe,EACf,MAAM,EAAE,UAAU,EAClB,SAAS,EACT,SAAS,GACV,EAAE;IACD,MAAM,EAAE,YAAY,CAAA;IACpB,UAAU,EAAE,MAAM,CAAA;IAClB,SAAS,EAAE,MAAM,CAAA;IACjB,eAAe,EAAE,MAAM,CAAA;IACvB,MAAM,EAAE,EAAE,CAAC,cAAc,EAAE,CAAA;IAC3B,SAAS,EAAE;QAAE,IAAI,EAAE,MAAM,CAAA;KAAE,CAAA;IAC3B,SAAS,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,CAAA;CACrC,GAAG,OAAO,CAAC,EAAE,CAAC,UAAU,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,CAwD9C;AA2HD,wBAAsB,YAAY,CAAC,EACjC,OAAO,EACP,GAAG,EACH,MAAM,EACN,WAAW,EACX,SAAS,EACT,MAAM,GACP,EAAE,yBAAyB,GAAG,OAAO,CAAC,IAAI,CAAC,CAsF3C"}
|
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
import { taskLog } from '@clack/prompts';
|
|
2
|
+
import { PullError } from '@filoz/synapse-core/errors';
|
|
2
3
|
import * as SP from '@filoz/synapse-core/sp';
|
|
4
|
+
import { signAddPieces } from '@filoz/synapse-core/typed-data';
|
|
3
5
|
import { and, asc, eq, gt, inArray } from 'drizzle-orm';
|
|
4
6
|
import PQueue from 'p-queue';
|
|
5
7
|
import { listDataSetPieces } from "../api.js";
|
|
@@ -10,8 +12,61 @@ import { upsertOperations } from "../db/upsert-operations.js";
|
|
|
10
12
|
import { completeConfirmedOperations, excludeOperationsByCid, fitAddPiecesBatch, hashLink, operationsToAddPieces, operationsToPullPieces, } from "../utils.js";
|
|
11
13
|
const MAX_BATCH_CANDIDATES = 500;
|
|
12
14
|
const MAX_PULL_PIECES = 108;
|
|
15
|
+
const MIN_PULL_PIECES = 10;
|
|
13
16
|
const PULL_RETRY_COUNT = 10;
|
|
14
|
-
function
|
|
17
|
+
function isPullBackpressure(error) {
|
|
18
|
+
return PullError.is(error) && error.details?.includes('pull queue backpressure') === true;
|
|
19
|
+
}
|
|
20
|
+
export async function pullInChunks({ client, serviceURL, dataSetId, clientDataSetId, pieces: pullPieces, pullChunk, onMessage, }) {
|
|
21
|
+
const results = [];
|
|
22
|
+
let offset = 0;
|
|
23
|
+
while (offset < pullPieces.length) {
|
|
24
|
+
const pieces = pullPieces.slice(offset, offset + pullChunk.size);
|
|
25
|
+
const extraData = await signAddPieces(client, {
|
|
26
|
+
clientDataSetId,
|
|
27
|
+
pieces: pieces.map(({ pieceCid }) => ({ pieceCid })),
|
|
28
|
+
});
|
|
29
|
+
if (pieces.length > pullChunk.size) {
|
|
30
|
+
continue;
|
|
31
|
+
}
|
|
32
|
+
if (pieces.length > MIN_PULL_PIECES) {
|
|
33
|
+
try {
|
|
34
|
+
await SP.pullPiecesApiRequest({
|
|
35
|
+
serviceURL,
|
|
36
|
+
recordKeeper: client.chain.contracts.fwss.address,
|
|
37
|
+
extraData,
|
|
38
|
+
dataSetId,
|
|
39
|
+
pieces: pieces.map(({ pieceCid, sourceUrl }) => ({ pieceCid: pieceCid.toString(), sourceUrl })),
|
|
40
|
+
retryCount: 0,
|
|
41
|
+
});
|
|
42
|
+
}
|
|
43
|
+
catch (error) {
|
|
44
|
+
if (isPullBackpressure(error)) {
|
|
45
|
+
pullChunk.size = Math.max(MIN_PULL_PIECES, Math.floor(Math.min(pullChunk.size, pieces.length) / 2));
|
|
46
|
+
onMessage(`Curio pull queue is busy, pulling ${pullChunk.size} pieces at a time`);
|
|
47
|
+
continue;
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
const pullResult = await SP.waitForPullPieces(client, {
|
|
52
|
+
serviceURL,
|
|
53
|
+
dataSetId,
|
|
54
|
+
clientDataSetId,
|
|
55
|
+
pieces,
|
|
56
|
+
extraData,
|
|
57
|
+
retryCount: PULL_RETRY_COUNT,
|
|
58
|
+
onStatus: (status) => {
|
|
59
|
+
const completed = status.pieces.filter((piece) => piece.status === 'complete').length;
|
|
60
|
+
const failed = status.pieces.filter((piece) => piece.status === 'failed').length;
|
|
61
|
+
onMessage(`Pulling pieces ${offset + 1}-${offset + pieces.length} of ${pullPieces.length}: ${completed} completed, ${failed} failed`);
|
|
62
|
+
},
|
|
63
|
+
});
|
|
64
|
+
results.push(...pullResult.pieces);
|
|
65
|
+
offset += pieces.length;
|
|
66
|
+
}
|
|
67
|
+
return results;
|
|
68
|
+
}
|
|
69
|
+
function createAddPiecesWorker({ localDb, targetCids, pullChunk, repair, client, state, log, }) {
|
|
15
70
|
return async (options) => {
|
|
16
71
|
let completedOps = 0;
|
|
17
72
|
let failedOps = 0;
|
|
@@ -31,38 +86,30 @@ function createAddPiecesWorker({ localDb, targetCids, pullChunkSize, repair, cli
|
|
|
31
86
|
}
|
|
32
87
|
group.message(`Pulling ${operations.length} pieces...`);
|
|
33
88
|
if (operations.length > 0) {
|
|
34
|
-
const
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
},
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
localDb,
|
|
59
|
-
operations: operationToFailed.map((operation) => ({
|
|
60
|
-
...operation,
|
|
61
|
-
status: 'failed',
|
|
62
|
-
error: `pull failed with status ${status}`,
|
|
63
|
-
})),
|
|
64
|
-
});
|
|
65
|
-
}
|
|
89
|
+
const pullResult = await pullInChunks({
|
|
90
|
+
client,
|
|
91
|
+
serviceURL: repair.targetProviderUrl,
|
|
92
|
+
dataSetId: dataset.dataSetId,
|
|
93
|
+
clientDataSetId: dataset.clientDataSetId,
|
|
94
|
+
pieces: operationsToPullPieces(operations),
|
|
95
|
+
pullChunk,
|
|
96
|
+
onMessage: (message) => group.message(message),
|
|
97
|
+
});
|
|
98
|
+
for (const { pieceCid, status } of pullResult) {
|
|
99
|
+
const cid = pieceCid.toString();
|
|
100
|
+
if (status !== 'complete') {
|
|
101
|
+
state.failedOperations++;
|
|
102
|
+
failedOps++;
|
|
103
|
+
const { operationToFailed, operationToCommit } = excludeOperationsByCid(operations, cid);
|
|
104
|
+
operations = operationToCommit;
|
|
105
|
+
await upsertOperations({
|
|
106
|
+
localDb,
|
|
107
|
+
operations: operationToFailed.map((operation) => ({
|
|
108
|
+
...operation,
|
|
109
|
+
status: 'failed',
|
|
110
|
+
error: `pull failed with status ${status}`,
|
|
111
|
+
})),
|
|
112
|
+
});
|
|
66
113
|
}
|
|
67
114
|
}
|
|
68
115
|
}
|
|
@@ -156,8 +203,8 @@ export async function runAddPieces({ localDb, api, repair, concurrency, batchSiz
|
|
|
156
203
|
cursor = operations.at(-1)?.id ?? cursor;
|
|
157
204
|
return operations;
|
|
158
205
|
}
|
|
159
|
-
const
|
|
160
|
-
const addPiecesJob = createAddPiecesWorker({ localDb, targetCids,
|
|
206
|
+
const pullChunk = { size: Math.max(1, Math.floor(MAX_PULL_PIECES / concurrency)) };
|
|
207
|
+
const addPiecesJob = createAddPiecesWorker({ localDb, targetCids, pullChunk, repair, client, state, log });
|
|
161
208
|
const queue = new PQueue({ concurrency });
|
|
162
209
|
while (true) {
|
|
163
210
|
await queue.onSizeLessThan(concurrency);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"add-pieces.js","sourceRoot":"","sources":["../../../src/pipeline/add-pieces.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,gBAAgB,CAAA;AACxC,OAAO,KAAK,EAAE,MAAM,wBAAwB,CAAA;AAC5C,OAAO,EAAE,GAAG,EAAE,GAAG,EAAE,EAAE,EAAE,EAAE,EAAE,OAAO,EAAE,MAAM,aAAa,CAAA;AACvD,OAAO,MAAM,MAAM,SAAS,CAAA;AAC5B,OAAO,EAAE,iBAAiB,EAAE,MAAM,WAAW,CAAA;AAC7C,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAA;AACjD,OAAO,EAAE,gBAAgB,EAAE,MAAM,6BAA6B,CAAA;AAC9D,OAAO,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAA;AACrD,OAAO,EAAE,gBAAgB,EAAE,MAAM,4BAA4B,CAAA;AAG7D,OAAO,EACL,2BAA2B,EAC3B,sBAAsB,EACtB,iBAAiB,EACjB,QAAQ,EACR,qBAAqB,EACrB,sBAAsB,GACvB,MAAM,aAAa,CAAA;AAGpB,MAAM,oBAAoB,GAAG,GAAG,CAAA;AAOhC,MAAM,eAAe,GAAG,GAAG,CAAA;
|
|
1
|
+
{"version":3,"file":"add-pieces.js","sourceRoot":"","sources":["../../../src/pipeline/add-pieces.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,gBAAgB,CAAA;AACxC,OAAO,EAAE,SAAS,EAAE,MAAM,4BAA4B,CAAA;AACtD,OAAO,KAAK,EAAE,MAAM,wBAAwB,CAAA;AAC5C,OAAO,EAAE,aAAa,EAAE,MAAM,gCAAgC,CAAA;AAC9D,OAAO,EAAE,GAAG,EAAE,GAAG,EAAE,EAAE,EAAE,EAAE,EAAE,OAAO,EAAE,MAAM,aAAa,CAAA;AACvD,OAAO,MAAM,MAAM,SAAS,CAAA;AAC5B,OAAO,EAAE,iBAAiB,EAAE,MAAM,WAAW,CAAA;AAC7C,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAA;AACjD,OAAO,EAAE,gBAAgB,EAAE,MAAM,6BAA6B,CAAA;AAC9D,OAAO,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAA;AACrD,OAAO,EAAE,gBAAgB,EAAE,MAAM,4BAA4B,CAAA;AAG7D,OAAO,EACL,2BAA2B,EAC3B,sBAAsB,EACtB,iBAAiB,EACjB,QAAQ,EACR,qBAAqB,EACrB,sBAAsB,GACvB,MAAM,aAAa,CAAA;AAGpB,MAAM,oBAAoB,GAAG,GAAG,CAAA;AAOhC,MAAM,eAAe,GAAG,GAAG,CAAA;AAM3B,MAAM,eAAe,GAAG,EAAE,CAAA;AAG1B,MAAM,gBAAgB,GAAG,EAAE,CAAA;AAG3B,SAAS,kBAAkB,CAAC,KAAc;IACxC,OAAO,SAAS,CAAC,EAAE,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,OAAO,EAAE,QAAQ,CAAC,yBAAyB,CAAC,KAAK,IAAI,CAAA;AAC3F,CAAC;AAqCD,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,EACjC,MAAM,EACN,UAAU,EACV,SAAS,EACT,eAAe,EACf,MAAM,EAAE,UAAU,EAClB,SAAS,EACT,SAAS,GASV;IACC,MAAM,OAAO,GAAuC,EAAE,CAAA;IACtD,IAAI,MAAM,GAAG,CAAC,CAAA;IACd,OAAO,MAAM,GAAG,UAAU,CAAC,MAAM,EAAE,CAAC;QAClC,MAAM,MAAM,GAAG,UAAU,CAAC,KAAK,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC,IAAI,CAAC,CAAA;QAEhE,MAAM,SAAS,GAAG,MAAM,aAAa,CAAC,MAAM,EAAE;YAC5C,eAAe;YACf,MAAM,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC;SACrD,CAAC,CAAA;QAEF,IAAI,MAAM,CAAC,MAAM,GAAG,SAAS,CAAC,IAAI,EAAE,CAAC;YACnC,SAAQ;QACV,CAAC;QAGD,IAAI,MAAM,CAAC,MAAM,GAAG,eAAe,EAAE,CAAC;YACpC,IAAI,CAAC;gBAEH,MAAM,EAAE,CAAC,oBAAoB,CAAC;oBAC5B,UAAU;oBACV,YAAY,EAAE,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO;oBACjD,SAAS;oBACT,SAAS;oBACT,MAAM,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,EAAE,SAAS,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,QAAQ,EAAE,QAAQ,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,CAAC,CAAC;oBAC/F,UAAU,EAAE,CAAC;iBACd,CAAC,CAAA;YACJ,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,IAAI,kBAAkB,CAAC,KAAK,CAAC,EAAE,CAAC;oBAC9B,SAAS,CAAC,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,eAAe,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,EAAE,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAA;oBACnG,SAAS,CAAC,qCAAqC,SAAS,CAAC,IAAI,mBAAmB,CAAC,CAAA;oBACjF,SAAQ;gBACV,CAAC;YAEH,CAAC;QACH,CAAC;QAED,MAAM,UAAU,GAAG,MAAM,EAAE,CAAC,iBAAiB,CAAC,MAAM,EAAE;YACpD,UAAU;YACV,SAAS;YACT,eAAe;YACf,MAAM;YACN,SAAS;YACT,UAAU,EAAE,gBAAgB;YAC5B,QAAQ,EAAE,CAAC,MAAM,EAAE,EAAE;gBACnB,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,KAAK,UAAU,CAAC,CAAC,MAAM,CAAA;gBACrF,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,MAAM,CAAA;gBAChF,SAAS,CACP,kBAAkB,MAAM,GAAG,CAAC,IAAI,MAAM,GAAG,MAAM,CAAC,MAAM,OAAO,UAAU,CAAC,MAAM,KAAK,SAAS,eAAe,MAAM,SAAS,CAC3H,CAAA;YACH,CAAC;SACF,CAAC,CAAA;QACF,OAAO,CAAC,IAAI,CAAC,GAAG,UAAU,CAAC,MAAM,CAAC,CAAA;QAClC,MAAM,IAAI,MAAM,CAAC,MAAM,CAAA;IACzB,CAAC;IACD,OAAO,OAAO,CAAA;AAChB,CAAC;AAKD,SAAS,qBAAqB,CAAC,EAC7B,OAAO,EACP,UAAU,EACV,SAAS,EACT,MAAM,EACN,MAAM,EACN,KAAK,EACL,GAAG,GAC0B;IAC7B,OAAO,KAAK,EAAE,OAA4B,EAAE,EAAE;QAC5C,IAAI,YAAY,GAAG,CAAC,CAAA;QACpB,IAAI,SAAS,GAAG,CAAC,CAAA;QACjB,IAAI,UAAU,GAAsB,OAAO,CAAC,UAAU,CAAA;QAEtD,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC,SAAS,OAAO,CAAC,WAAW,KAAK,OAAO,CAAC,UAAU,CAAC,MAAM,UAAU,CAAC,CAAA;QAE7F,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,MAAM,gBAAgB,CAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,CAAC,EAAE,EAAE,MAAM,EAAE,CAAC,CAAA;YAGhF,IAAI,UAAU,EAAE,CAAC;gBACf,UAAU,GAAG,MAAM,UAAU,CAAC,EAAE,OAAO,EAAE,YAAY,EAAE,UAAU,EAAE,UAAU,EAAE,CAAC,CAAA;YAClF,CAAC;YAED,MAAM,iCAAiC,GAAG,UAAU,CAAC,MAAM,CAAA;YAC3D,UAAU,GAAG,MAAM,2BAA2B,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC,CAAA;YAC/E,MAAM,mBAAmB,GAAG,iCAAiC,GAAG,UAAU,CAAC,MAAM,CAAA;YACjF,IAAI,mBAAmB,GAAG,CAAC,EAAE,CAAC;gBAC5B,KAAK,CAAC,mBAAmB,IAAI,mBAAmB,CAAA;gBAChD,YAAY,IAAI,mBAAmB,CAAA;YACrC,CAAC;YAED,KAAK,CAAC,OAAO,CAAC,WAAW,UAAU,CAAC,MAAM,YAAY,CAAC,CAAA;YAEvD,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC1B,MAAM,UAAU,GAAG,MAAM,YAAY,CAAC;oBACpC,MAAM;oBACN,UAAU,EAAE,MAAM,CAAC,iBAAiB;oBACpC,SAAS,EAAE,OAAO,CAAC,SAAS;oBAC5B,eAAe,EAAE,OAAO,CAAC,eAAe;oBACxC,MAAM,EAAE,sBAAsB,CAAC,UAAU,CAAC;oBAC1C,SAAS;oBACT,SAAS,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC;iBAC/C,CAAC,CAAA;gBAEF,KAAK,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,UAAU,EAAE,CAAC;oBAC9C,MAAM,GAAG,GAAG,QAAQ,CAAC,QAAQ,EAAE,CAAA;oBAC/B,IAAI,MAAM,KAAK,UAAU,EAAE,CAAC;wBAC1B,KAAK,CAAC,gBAAgB,EAAE,CAAA;wBACxB,SAAS,EAAE,CAAA;wBACX,MAAM,EAAE,iBAAiB,EAAE,iBAAiB,EAAE,GAAG,sBAAsB,CAAC,UAAU,EAAE,GAAG,CAAC,CAAA;wBACxF,UAAU,GAAG,iBAAiB,CAAA;wBAC9B,MAAM,gBAAgB,CAAC;4BACrB,OAAO;4BACP,UAAU,EAAE,iBAAiB,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;gCAChD,GAAG,SAAS;gCACZ,MAAM,EAAE,QAAQ;gCAChB,KAAK,EAAE,2BAA2B,MAAM,EAAE;6BAC3C,CAAC,CAAC;yBACJ,CAAC,CAAA;oBACJ,CAAC;gBACH,CAAC;YACH,CAAC;YAGD,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC1B,KAAK,CAAC,OAAO,CAAC,UAAU,UAAU,CAAC,MAAM,YAAY,CAAC,CAAA;gBACtD,MAAM,eAAe,GAAG,MAAM,EAAE,CAAC,SAAS,CAAC,MAAM,EAAE;oBACjD,UAAU,EAAE,MAAM,CAAC,iBAAiB;oBACpC,SAAS,EAAE,OAAO,CAAC,SAAS;oBAC5B,eAAe,EAAE,OAAO,CAAC,eAAe;oBACxC,MAAM,EAAE,qBAAqB,CAAC,UAAU,EAAE,CAAC,UAAU,CAAC;iBACvD,CAAC,CAAA;gBACF,MAAM,gBAAgB,CAAC;oBACrB,OAAO;oBACP,UAAU,EAAE,UAAU,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;wBACzC,GAAG,SAAS;wBACZ,MAAM,EAAE,eAAe,CAAC,MAAM;qBAC/B,CAAC,CAAC;iBACJ,CAAC,CAAA;gBAEF,KAAK,CAAC,OAAO,CAAC,0BAA0B,QAAQ,CAAC,eAAe,CAAC,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAA;gBAC5F,MAAM,gBAAgB,GAAG,MAAM,EAAE,CAAC,gBAAgB,CAAC,eAAe,CAAC,CAAA;gBAEnE,MAAM,eAAe,GAAG,gBAAgB,CAAC,eAAe,IAAI,gBAAgB,CAAC,MAAM,CAAA;gBACnF,KAAK,CAAC,mBAAmB,IAAI,UAAU,CAAC,MAAM,CAAA;gBAC9C,YAAY,IAAI,UAAU,CAAC,MAAM,CAAA;gBACjC,KAAK,MAAM,SAAS,IAAI,UAAU;oBAAE,UAAU,EAAE,GAAG,CAAC,SAAS,CAAC,GAAG,CAAC,CAAA;gBAClE,MAAM,gBAAgB,CAAC;oBACrB,OAAO;oBACP,UAAU,EAAE,UAAU,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;wBACzC,GAAG,SAAS;wBACZ,MAAM,EAAE,WAAW;wBACnB,KAAK,EAAE,IAAI;wBACX,MAAM,EAAE,eAAe;qBACxB,CAAC,CAAC;iBACJ,CAAC,CAAA;YACJ,CAAC;YACD,KAAK,CAAC,OAAO,CAAC,SAAS,YAAY,WAAW,SAAS,SAAS,CAAC,CAAA;QACnE,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,KAAK,CAAC,gBAAgB,IAAI,UAAU,CAAC,MAAM,CAAA;YAC3C,MAAM,OAAO,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,CAAA;YACxE,KAAK,CAAC,OAAO,CAAC,GAAG,OAAO,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,EAAE,CAAC,CAAA;YAC/C,MAAM,gBAAgB,CAAC;gBACrB,OAAO;gBACP,UAAU,EAAE,UAAU,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;oBACzC,GAAG,SAAS;oBACZ,MAAM,EAAE,QAAQ;oBAChB,KAAK,EAAE,OAAO;iBACf,CAAC,CAAC;aACJ,CAAC,CAAA;QACJ,CAAC;IACH,CAAC,CAAA;AACH,CAAC;AAKD,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,EACjC,OAAO,EACP,GAAG,EACH,MAAM,EACN,WAAW,EACX,SAAS,EACT,MAAM,GACoB;IAC1B,MAAM,WAAW,GAAG,OAAO,CAAC,CAAC,CAAC,UAAU,CAAA;IACxC,IAAI,MAAM,GAAG,CAAC,CAAA;IAEd,MAAM,eAAe,GAAG,MAAM,OAAO,CAAC,MAAM,CAC1C,WAAW,CAAC,UAAU,EACtB,GAAG,CACD,EAAE,CAAC,WAAW,CAAC,UAAU,CAAC,QAAQ,EAAE,MAAM,CAAC,EAAE,CAAC,EAC9C,EAAE,CAAC,WAAW,CAAC,UAAU,CAAC,IAAI,EAAE,WAAW,CAAC,EAC5C,OAAO,CAAC,WAAW,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC,CAC9D,CACF,CAAA;IACD,IAAI,WAAW,GAAG,CAAC,CAAA;IACnB,MAAM,KAAK,GAAG;QACZ,eAAe;QACf,mBAAmB,EAAE,CAAC;QACtB,gBAAgB,EAAE,CAAC;KACpB,CAAA;IAED,MAAM,GAAG,GAAG,OAAO,CAAC;QAClB,KAAK,EAAE,eAAe;QACtB,KAAK,EAAE,CAAC;KACT,CAAC,CAAA;IAEF,MAAM,OAAO,GAAG,eAAe,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,gBAAgB,CAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,CAAC,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC,SAAS,CAAA;IAGlH,IAAI,UAAmC,CAAA;IACvC,IAAI,MAAM,CAAC,eAAe,IAAI,IAAI,IAAI,OAAO,EAAE,CAAC;QAC9C,MAAM,MAAM,GAAG,MAAM,iBAAiB,CAAC,GAAG,EAAE,OAAO,CAAC,SAAS,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAA;QAClF,UAAU,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,IAAI,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAA;IACzF,CAAC;IAED,KAAK,UAAU,YAAY;QACzB,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,OAAO,IAAI,CAAA;QACb,CAAC;QACD,MAAM,UAAU,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,UAAU,CAAC,QAAQ,CAAC;YACzD,KAAK,EAAE,GAAG,CACR,EAAE,CAAC,WAAW,CAAC,UAAU,CAAC,QAAQ,EAAE,MAAM,CAAC,EAAE,CAAC,EAC9C,EAAE,CAAC,WAAW,CAAC,UAAU,CAAC,IAAI,EAAE,WAAW,CAAC,EAC5C,OAAO,CAAC,WAAW,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC,EAC7D,EAAE,CAAC,WAAW,CAAC,UAAU,CAAC,EAAE,EAAE,MAAM,CAAC,CACtC;YACD,OAAO,EAAE,CAAC,GAAG,CAAC,WAAW,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;YACzC,KAAK,EAAE,SAAS,IAAI,oBAAoB;SACzC,CAAC,CAAA;QACF,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC5B,OAAO,IAAI,CAAA;QACb,CAAC;QAGD,MAAM,UAAU,GAAG,iBAAiB,CAAC;YACnC,UAAU,EAAE,UAAU;YACtB,SAAS,EAAE,OAAO,CAAC,SAAS;YAC5B,KAAK,EAAE,MAAM,CAAC,KAAK;YACnB,YAAY,EAAE,CAAC,UAAU;SAC1B,CAAC,CAAA;QAEF,MAAM,GAAG,UAAU,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,IAAI,MAAM,CAAA;QACxC,OAAO,UAAU,CAAA;IACnB,CAAC;IAGD,MAAM,SAAS,GAAG,EAAE,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,eAAe,GAAG,WAAW,CAAC,CAAC,EAAE,CAAA;IAClF,MAAM,YAAY,GAAG,qBAAqB,CAAC,EAAE,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,CAAA;IAC1G,MAAM,KAAK,GAAG,IAAI,MAAM,CAAC,EAAE,WAAW,EAAE,CAAC,CAAA;IAEzC,OAAO,IAAI,EAAE,CAAC;QACZ,MAAM,KAAK,CAAC,cAAc,CAAC,WAAW,CAAC,CAAA;QACvC,MAAM,UAAU,GAAG,MAAM,YAAY,EAAE,CAAA;QACvC,IAAI,CAAC,UAAU;YAAE,MAAK;QACtB,WAAW,EAAE,CAAA;QACb,MAAM,kBAAkB,GAAG,WAAW,CAAA;QACtC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,YAAY,CAAC,EAAE,UAAU,EAAE,WAAW,EAAE,kBAAkB,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;IACrG,CAAC;IAED,MAAM,KAAK,CAAC,MAAM,EAAE,CAAA;IAEpB,GAAG,CAAC,OAAO,CAAC,SAAS,KAAK,CAAC,mBAAmB,YAAY,KAAK,CAAC,gBAAgB,SAAS,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAA;IAE7G,MAAM,YAAY,CAAC;QACjB,OAAO;QACP,QAAQ,EAAE,MAAM,CAAC,EAAE;QACnB,MAAM,EAAE,KAAK,CAAC,gBAAgB,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,WAAW;KAC5D,CAAC,CAAA;AACJ,CAAC"}
|
package/package.json
CHANGED
package/readme.md
CHANGED
|
@@ -242,7 +242,7 @@ The command first ensures the target repair dataset exists for the configured wa
|
|
|
242
242
|
|
|
243
243
|
Options:
|
|
244
244
|
|
|
245
|
-
- `--concurrency <number>` controls how many batches run at once. Defaults to `4`. Each batch pulls its pieces in chunks so that no more than 108 pieces are pulling at once across all batches, which is Curio's per-client limit. With `4`, each pull request holds up to 27 pieces.
|
|
245
|
+
- `--concurrency <number>` controls how many batches run at once. Defaults to `4`. Each batch pulls its pieces in chunks so that no more than 108 pieces are pulling at once across all batches, which is Curio's per-client limit. With `4`, each pull request holds up to 27 pieces. Curio lowers that limit while other clients have pulls pending; when it refuses a pull, the CLI halves the chunk size (down to 10 pieces) for the rest of the run.
|
|
246
246
|
- `--batch-size <number>` caps the number of `add_piece` operations per batch. By default each batch holds as many pieces as fit in one `addPieces` message (about 400 without piece metadata, 80 for legacy datasets).
|
|
247
247
|
- `--payer <address>` overrides the payer address used when creating or finding the target repair dataset. Defaults to the configured wallet address.
|
|
248
248
|
|
|
@@ -306,7 +306,7 @@ The command creates a new target dataset on the target provider, copying CDN and
|
|
|
306
306
|
|
|
307
307
|
Options:
|
|
308
308
|
|
|
309
|
-
- `--concurrency <number>` controls how many batches run at once. Defaults to `4`. Each batch pulls its pieces in chunks so that no more than 108 pieces are pulling at once across all batches, which is Curio's per-client limit. With `4`, each pull request holds up to 27 pieces.
|
|
309
|
+
- `--concurrency <number>` controls how many batches run at once. Defaults to `4`. Each batch pulls its pieces in chunks so that no more than 108 pieces are pulling at once across all batches, which is Curio's per-client limit. With `4`, each pull request holds up to 27 pieces. Curio lowers that limit while other clients have pulls pending; when it refuses a pull, the CLI halves the chunk size (down to 10 pieces) for the rest of the run.
|
|
310
310
|
- `--batch-size <number>` caps the number of `add_piece` operations per batch. By default each batch holds as many pieces as fit in one `addPieces` message (about 400 without piece metadata, 80 for legacy datasets).
|
|
311
311
|
- `--payer <address>` overrides the payer address used when creating the target replication dataset. Defaults to the configured wallet address.
|
|
312
312
|
|
|
@@ -375,7 +375,7 @@ The command first ensures the target dataset exists for the configured wallet an
|
|
|
375
375
|
|
|
376
376
|
Options:
|
|
377
377
|
|
|
378
|
-
- `--concurrency <number>` controls how many batches run at once. Defaults to `4`. Each batch pulls its pieces in chunks so that no more than 108 pieces are pulling at once across all batches, which is Curio's per-client limit. With `4`, each pull request holds up to 27 pieces.
|
|
378
|
+
- `--concurrency <number>` controls how many batches run at once. Defaults to `4`. Each batch pulls its pieces in chunks so that no more than 108 pieces are pulling at once across all batches, which is Curio's per-client limit. With `4`, each pull request holds up to 27 pieces. Curio lowers that limit while other clients have pulls pending; when it refuses a pull, the CLI halves the chunk size (down to 10 pieces) for the rest of the run.
|
|
379
379
|
- `--batch-size <number>` caps the number of `add_piece` operations per batch. By default each batch holds as many pieces as fit in one `addPieces` message (about 400 without piece metadata, 80 for legacy datasets).
|
|
380
380
|
- `--payer <address>` overrides the payer address used when creating or finding the target dataset. Defaults to the configured wallet address.
|
|
381
381
|
|
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
import { taskLog } from '@clack/prompts'
|
|
2
|
+
import { PullError } from '@filoz/synapse-core/errors'
|
|
2
3
|
import * as SP from '@filoz/synapse-core/sp'
|
|
4
|
+
import { signAddPieces } from '@filoz/synapse-core/typed-data'
|
|
3
5
|
import { and, asc, eq, gt, inArray } from 'drizzle-orm'
|
|
4
6
|
import PQueue from 'p-queue'
|
|
5
7
|
import { listDataSetPieces } from '../api.ts'
|
|
@@ -28,9 +30,20 @@ const MAX_BATCH_CANDIDATES = 500
|
|
|
28
30
|
*/
|
|
29
31
|
const MAX_PULL_PIECES = 108
|
|
30
32
|
|
|
33
|
+
/**
|
|
34
|
+
* Other clients' pending pieces lower our per-client limit, but never below 10.
|
|
35
|
+
* Pulls this small are refused only when the whole node is full.
|
|
36
|
+
*/
|
|
37
|
+
const MIN_PULL_PIECES = 10
|
|
38
|
+
|
|
31
39
|
/** Curio answers 429 with Retry-After (1-5 minutes) while its pull queue is full; wait it out instead of failing. */
|
|
32
40
|
const PULL_RETRY_COUNT = 10
|
|
33
41
|
|
|
42
|
+
/** Curio's 429 body when a pull would exceed its pending-piece limits. */
|
|
43
|
+
function isPullBackpressure(error: unknown) {
|
|
44
|
+
return PullError.is(error) && error.details?.includes('pull queue backpressure') === true
|
|
45
|
+
}
|
|
46
|
+
|
|
34
47
|
export type RunPullPiecesPhaseOptions = {
|
|
35
48
|
localDb: LocalDatabase
|
|
36
49
|
api: IndexerApi
|
|
@@ -45,8 +58,8 @@ type CreateAddPiecesWorkerOptions = {
|
|
|
45
58
|
localDb: LocalDatabase
|
|
46
59
|
/** CIDs already in the target dataset; repairs only. Updated as batches are added. */
|
|
47
60
|
targetCids?: Set<string>
|
|
48
|
-
/** Max unique pieces per pull request. */
|
|
49
|
-
|
|
61
|
+
/** Max unique pieces per pull request, shared by all workers and lowered when Curio pushes back. */
|
|
62
|
+
pullChunk: { size: number }
|
|
50
63
|
repair: RepairSelect
|
|
51
64
|
client: WalletClient
|
|
52
65
|
state: {
|
|
@@ -62,13 +75,91 @@ type AddPiecesJobOptions = {
|
|
|
62
75
|
batchNumber: number
|
|
63
76
|
}
|
|
64
77
|
|
|
78
|
+
/**
|
|
79
|
+
* Pull pieces in chunks Curio admits, one chunk at a time.
|
|
80
|
+
* When Curio refuses a chunk for backpressure, halve `pullChunk.size` (shared with other workers) and try again.
|
|
81
|
+
*/
|
|
82
|
+
export async function pullInChunks({
|
|
83
|
+
client,
|
|
84
|
+
serviceURL,
|
|
85
|
+
dataSetId,
|
|
86
|
+
clientDataSetId,
|
|
87
|
+
pieces: pullPieces,
|
|
88
|
+
pullChunk,
|
|
89
|
+
onMessage,
|
|
90
|
+
}: {
|
|
91
|
+
client: WalletClient
|
|
92
|
+
serviceURL: string
|
|
93
|
+
dataSetId: bigint
|
|
94
|
+
clientDataSetId: bigint
|
|
95
|
+
pieces: SP.PullPieceInput[]
|
|
96
|
+
pullChunk: { size: number }
|
|
97
|
+
onMessage: (message: string) => void
|
|
98
|
+
}): Promise<SP.pullPieces.ReturnType['pieces']> {
|
|
99
|
+
const results: SP.pullPieces.ReturnType['pieces'] = []
|
|
100
|
+
let offset = 0
|
|
101
|
+
while (offset < pullPieces.length) {
|
|
102
|
+
const pieces = pullPieces.slice(offset, offset + pullChunk.size)
|
|
103
|
+
// Sign once so the admission check and the wait below share one pull.
|
|
104
|
+
const extraData = await signAddPieces(client, {
|
|
105
|
+
clientDataSetId,
|
|
106
|
+
pieces: pieces.map(({ pieceCid }) => ({ pieceCid })),
|
|
107
|
+
})
|
|
108
|
+
// Another worker may have shrunk the shared size while this chunk was signed; re-slice to match.
|
|
109
|
+
if (pieces.length > pullChunk.size) {
|
|
110
|
+
continue
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
// Another client's pending pull lowers our limit; shrink the chunk instead of waiting on the same size.
|
|
114
|
+
if (pieces.length > MIN_PULL_PIECES) {
|
|
115
|
+
try {
|
|
116
|
+
// Single POST: the SDK's own retries would wait out Retry-After on the same size.
|
|
117
|
+
await SP.pullPiecesApiRequest({
|
|
118
|
+
serviceURL,
|
|
119
|
+
recordKeeper: client.chain.contracts.fwss.address,
|
|
120
|
+
extraData,
|
|
121
|
+
dataSetId,
|
|
122
|
+
pieces: pieces.map(({ pieceCid, sourceUrl }) => ({ pieceCid: pieceCid.toString(), sourceUrl })),
|
|
123
|
+
retryCount: 0,
|
|
124
|
+
})
|
|
125
|
+
} catch (error) {
|
|
126
|
+
if (isPullBackpressure(error)) {
|
|
127
|
+
pullChunk.size = Math.max(MIN_PULL_PIECES, Math.floor(Math.min(pullChunk.size, pieces.length) / 2))
|
|
128
|
+
onMessage(`Curio pull queue is busy, pulling ${pullChunk.size} pieces at a time`)
|
|
129
|
+
continue
|
|
130
|
+
}
|
|
131
|
+
// Leave other errors to the retries in waitForPullPieces.
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
const pullResult = await SP.waitForPullPieces(client, {
|
|
136
|
+
serviceURL,
|
|
137
|
+
dataSetId,
|
|
138
|
+
clientDataSetId,
|
|
139
|
+
pieces,
|
|
140
|
+
extraData,
|
|
141
|
+
retryCount: PULL_RETRY_COUNT,
|
|
142
|
+
onStatus: (status) => {
|
|
143
|
+
const completed = status.pieces.filter((piece) => piece.status === 'complete').length
|
|
144
|
+
const failed = status.pieces.filter((piece) => piece.status === 'failed').length
|
|
145
|
+
onMessage(
|
|
146
|
+
`Pulling pieces ${offset + 1}-${offset + pieces.length} of ${pullPieces.length}: ${completed} completed, ${failed} failed`
|
|
147
|
+
)
|
|
148
|
+
},
|
|
149
|
+
})
|
|
150
|
+
results.push(...pullResult.pieces)
|
|
151
|
+
offset += pieces.length
|
|
152
|
+
}
|
|
153
|
+
return results
|
|
154
|
+
}
|
|
155
|
+
|
|
65
156
|
/**
|
|
66
157
|
* Create a worker function that adds pieces to the target repair dataset.
|
|
67
158
|
*/
|
|
68
159
|
function createAddPiecesWorker({
|
|
69
160
|
localDb,
|
|
70
161
|
targetCids,
|
|
71
|
-
|
|
162
|
+
pullChunk,
|
|
72
163
|
repair,
|
|
73
164
|
client,
|
|
74
165
|
state,
|
|
@@ -100,39 +191,31 @@ function createAddPiecesWorker({
|
|
|
100
191
|
group.message(`Pulling ${operations.length} pieces...`)
|
|
101
192
|
// pull pieces in chunks that Curio admits, one chunk at a time per worker
|
|
102
193
|
if (operations.length > 0) {
|
|
103
|
-
const
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
localDb,
|
|
129
|
-
operations: operationToFailed.map((operation) => ({
|
|
130
|
-
...operation,
|
|
131
|
-
status: 'failed',
|
|
132
|
-
error: `pull failed with status ${status}`,
|
|
133
|
-
})),
|
|
134
|
-
})
|
|
135
|
-
}
|
|
194
|
+
const pullResult = await pullInChunks({
|
|
195
|
+
client,
|
|
196
|
+
serviceURL: repair.targetProviderUrl,
|
|
197
|
+
dataSetId: dataset.dataSetId,
|
|
198
|
+
clientDataSetId: dataset.clientDataSetId,
|
|
199
|
+
pieces: operationsToPullPieces(operations),
|
|
200
|
+
pullChunk,
|
|
201
|
+
onMessage: (message) => group.message(message),
|
|
202
|
+
})
|
|
203
|
+
|
|
204
|
+
for (const { pieceCid, status } of pullResult) {
|
|
205
|
+
const cid = pieceCid.toString()
|
|
206
|
+
if (status !== 'complete') {
|
|
207
|
+
state.failedOperations++
|
|
208
|
+
failedOps++
|
|
209
|
+
const { operationToFailed, operationToCommit } = excludeOperationsByCid(operations, cid)
|
|
210
|
+
operations = operationToCommit
|
|
211
|
+
await upsertOperations({
|
|
212
|
+
localDb,
|
|
213
|
+
operations: operationToFailed.map((operation) => ({
|
|
214
|
+
...operation,
|
|
215
|
+
status: 'failed',
|
|
216
|
+
error: `pull failed with status ${status}`,
|
|
217
|
+
})),
|
|
218
|
+
})
|
|
136
219
|
}
|
|
137
220
|
}
|
|
138
221
|
}
|
|
@@ -262,8 +345,8 @@ export async function runAddPieces({
|
|
|
262
345
|
}
|
|
263
346
|
|
|
264
347
|
// Keep pieces pulling across all workers within Curio's per-client limit.
|
|
265
|
-
const
|
|
266
|
-
const addPiecesJob = createAddPiecesWorker({ localDb, targetCids,
|
|
348
|
+
const pullChunk = { size: Math.max(1, Math.floor(MAX_PULL_PIECES / concurrency)) }
|
|
349
|
+
const addPiecesJob = createAddPiecesWorker({ localDb, targetCids, pullChunk, repair, client, state, log })
|
|
267
350
|
const queue = new PQueue({ concurrency })
|
|
268
351
|
|
|
269
352
|
while (true) {
|