@porulle/plugin-channel-connector 0.33.0 → 0.34.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +15 -0
- package/dist/index.js +132 -44
- package/package.json +4 -4
- package/src/index.ts +148 -31
package/dist/index.d.ts
CHANGED
|
@@ -6,6 +6,21 @@ export { isValidCatalogMappingFieldPath, matchFieldPath, mergeCatalogFieldMappin
|
|
|
6
6
|
export type { CatalogFieldMapping, CatalogFieldMappingInput, CatalogFieldMappingRow, CatalogFieldTarget, } from "./catalog-field-mapping.js";
|
|
7
7
|
/** ~320 Neon HTTP subrequests per product against a 10,000 per-invocation cap → hard ceiling near 31; 20 leaves margin for heavier products. */
|
|
8
8
|
export declare const CHANNEL_IMPORT_MAX_ITEMS_PER_INVOCATION = 20;
|
|
9
|
+
/**
|
|
10
|
+
* How many bounded batches one sweep may walk before it refuses rather than loops.
|
|
11
|
+
*
|
|
12
|
+
* A batched task walks its whole store inside ONE Workflow instance, so the ceiling that matters is
|
|
13
|
+
* the Workflows steps-per-instance limit: 10,000 on Workers Paid by default, raisable to 25,000
|
|
14
|
+
* (`limits.steps`), and 1,024 on Free. Every batch is one step. A 1,000-product merchant is roughly
|
|
15
|
+
* 650 inventory batches, so this bound is about eight times the largest real sweep and still an
|
|
16
|
+
* order of magnitude under the platform default — it exists to turn a cursor that stops advancing
|
|
17
|
+
* into a loud failure, not to ration normal work.
|
|
18
|
+
*
|
|
19
|
+
* It is NOT the limit that used to kill these sweeps. That was the request-chain depth of 32 Worker
|
|
20
|
+
* invocations, which a self-enqueueing continuation spends one of per batch; see the comment on the
|
|
21
|
+
* sweep loop below.
|
|
22
|
+
*/
|
|
23
|
+
export declare const CHANNEL_MAX_BATCHES_PER_SWEEP = 5000;
|
|
9
24
|
export { signState, verifyState } from "./oauth-state.js";
|
|
10
25
|
export type { BackfillCatalogOptions, BackfillCatalogReport, BuildCatalogPushItemsOptions, BuildCatalogPushItemsResult, CatalogPushAssemblyField, CatalogPushAssemblyImage, CatalogPushAssemblyItem, CatalogPushPreviewBefore, CatalogPushPreviewBeforeStatus, CatalogPushPreviewDiff, CatalogPushPreviewItem, CatalogPushPreviewResult, CatalogPushPreviewUnavailable, PushCatalogToStoreResult, CatalogPushJobResult, CatalogFieldConflict, CatalogFieldSkip, CatalogPushFieldSkip, CatalogPushSkipReason, CatalogConflictState, CatalogWriteSettings, ChannelComplianceData, ChannelConnectorPluginOptions, ChannelStockLine, ExportState, PublicConnectedStore, ReconcileReport, } from "./service.js";
|
|
11
26
|
export type { OAuthStatePayload, OAuthStateResult } from "./oauth-state.js";
|
package/dist/index.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { createHash } from "node:crypto";
|
|
2
|
-
import { CommerceConflictError, CommerceInvalidTransitionError, CommerceNotFoundError, CommerceValidationError, defineCommercePlugin, router, createSystemActor, isValidFieldPath, requireUserId, } from "@porulle/core";
|
|
2
|
+
import { CommerceConflictError, CommerceInvalidTransitionError, CommerceNotFoundError, CommerceValidationError, defineCommercePlugin, router, createSystemActor, isValidFieldPath, requireUserId, TaskNonRetryableError, } from "@porulle/core";
|
|
3
3
|
import { z } from "@hono/zod-openapi";
|
|
4
4
|
import { and, eq } from "@porulle/core/drizzle";
|
|
5
5
|
import { processedWebhookEvents } from "@porulle/core/schema";
|
|
@@ -12,6 +12,86 @@ export { ChannelConnectorService, CATALOG_OUTBOUND_SUPPRESSION_WINDOW_MS, CATALO
|
|
|
12
12
|
export { isValidCatalogMappingFieldPath, matchFieldPath, mergeCatalogFieldMapping, normalizeCatalogFieldMapping, compareCatalogFieldMappingSpecificity, providerCatalogFieldMappingDefaults, selectCatalogFieldMapping, validateCatalogMappingRow, } from "./catalog-field-mapping.js";
|
|
13
13
|
/** ~320 Neon HTTP subrequests per product against a 10,000 per-invocation cap → hard ceiling near 31; 20 leaves margin for heavier products. */
|
|
14
14
|
export const CHANNEL_IMPORT_MAX_ITEMS_PER_INVOCATION = 20;
|
|
15
|
+
/**
|
|
16
|
+
* How many bounded batches one sweep may walk before it refuses rather than loops.
|
|
17
|
+
*
|
|
18
|
+
* A batched task walks its whole store inside ONE Workflow instance, so the ceiling that matters is
|
|
19
|
+
* the Workflows steps-per-instance limit: 10,000 on Workers Paid by default, raisable to 25,000
|
|
20
|
+
* (`limits.steps`), and 1,024 on Free. Every batch is one step. A 1,000-product merchant is roughly
|
|
21
|
+
* 650 inventory batches, so this bound is about eight times the largest real sweep and still an
|
|
22
|
+
* order of magnitude under the platform default — it exists to turn a cursor that stops advancing
|
|
23
|
+
* into a loud failure, not to ration normal work.
|
|
24
|
+
*
|
|
25
|
+
* It is NOT the limit that used to kill these sweeps. That was the request-chain depth of 32 Worker
|
|
26
|
+
* invocations, which a self-enqueueing continuation spends one of per batch; see the comment on the
|
|
27
|
+
* sweep loop below.
|
|
28
|
+
*/
|
|
29
|
+
export const CHANNEL_MAX_BATCHES_PER_SWEEP = 5_000;
|
|
30
|
+
/**
|
|
31
|
+
* Walks a store's bounded batches to exhaustion INSIDE THE CALLING INSTANCE, one durable step per
|
|
32
|
+
* batch.
|
|
33
|
+
*
|
|
34
|
+
* WHY THIS SHAPE, in arithmetic rather than in adjectives. Each batch used to create its successor
|
|
35
|
+
* by calling `jobs.enqueue` from inside its own running Workflow instance. Cloudflare caps a single
|
|
36
|
+
* request chain at **32 Worker invocations** ("A single request has a maximum of 32 Worker
|
|
37
|
+
* invocations, and each call to a Service binding counts towards this limit" — Service bindings,
|
|
38
|
+
* Runtime APIs), and a continuation created inside its predecessor spends one, permanently. Two
|
|
39
|
+
* deaths on the deployed Worker on 2026-09-15, same error, same step (`porulle-turn:acquire:0`, the
|
|
40
|
+
* coordinator call, the chain's FIRST step):
|
|
41
|
+
*
|
|
42
|
+
* chain begun inside the catalog import -> died after 18 batches, at offset 360
|
|
43
|
+
* chain begun from a fetch handler -> died after 30 batches, at offset 960
|
|
44
|
+
*
|
|
45
|
+
* 18 + the ~14 the import had already spent ≈ 32; 30 + 2 ≈ 32. What varied was never the volume of
|
|
46
|
+
* work — it was the depth the chain STARTED at, which is why this read as an unreproducible "batch
|
|
47
|
+
* 7 one day, batch 37 the next" for three sessions. gflock-100 needs 65 inventory batches. **No
|
|
48
|
+
* chain survives a catalog of any real size at any batch size, and halving the batch doubles the
|
|
49
|
+
* chain**, which is why the intuitive fix is backwards.
|
|
50
|
+
*
|
|
51
|
+
* A `ctx.step.do` is not an invocation of another Worker. It spends no chain depth, so the walk
|
|
52
|
+
* below is flat however many batches it takes. Do not reintroduce an enqueue of the same slug here:
|
|
53
|
+
* that is the defect, and it looks like a one-line convenience.
|
|
54
|
+
*
|
|
55
|
+
* THE BUDGET PER BATCH GOT BIGGER, NOT SMALLER. A Workflow step's default timeout is 10 minutes
|
|
56
|
+
* (Workflows → Sleeping and retrying: limit 5, 10 s delay, exponential backoff, 10-minute timeout),
|
|
57
|
+
* which is exactly the budget the WHOLE sweep used to have — instance 1428d7e0 died on it with
|
|
58
|
+
* `WorkflowTimeoutError: Execution timed out after 600000ms`, having written 232 of ~1,299 levels.
|
|
59
|
+
* Each batch now gets that budget on its own, and a failed batch retries alone from the cursor its
|
|
60
|
+
* predecessor persisted rather than restarting the store.
|
|
61
|
+
*
|
|
62
|
+
* THE NAME IS LOAD-BEARING. The engine keys a step by its name and replays the cached result for a
|
|
63
|
+
* repeat, so a loop naming every step the same finishes instantly, reports success, and writes one
|
|
64
|
+
* batch. The batch index in the name is the only thing preventing that, and
|
|
65
|
+
* `batched-tasks-do-not-chain.test.ts` asserts the names are distinct.
|
|
66
|
+
*/
|
|
67
|
+
async function walkBatches(ctx, label, storeId, runBatch) {
|
|
68
|
+
let counted = 0;
|
|
69
|
+
let batches = 0;
|
|
70
|
+
const warnings = [];
|
|
71
|
+
let last = { exhausted: true, counted: 0, cursor: null };
|
|
72
|
+
// `ctx.step` is absent on engines that have not wired one (pg-boss, Inngest, Trigger). Running
|
|
73
|
+
// the batch inline there is the same walk without durability — which is exactly what the drizzle
|
|
74
|
+
// engine's own pass-through step does — so the plugin keeps working rather than throwing on a
|
|
75
|
+
// property it only needs for resumability.
|
|
76
|
+
const step = ctx.step ?? { do: (_name, fn) => fn() };
|
|
77
|
+
for (;;) {
|
|
78
|
+
// eslint-disable-next-line no-await-in-loop -- batches are sequential by construction: each one resumes from the cursor the previous one persisted.
|
|
79
|
+
last = await step.do(`${label}:${storeId}:batch:${batches}`, runBatch);
|
|
80
|
+
counted += last.counted;
|
|
81
|
+
if (last.warnings)
|
|
82
|
+
warnings.push(...last.warnings);
|
|
83
|
+
batches += 1;
|
|
84
|
+
if (last.exhausted)
|
|
85
|
+
return { counted, batches, last, warnings };
|
|
86
|
+
if (batches >= CHANNEL_MAX_BATCHES_PER_SWEEP) {
|
|
87
|
+
// A cursor that stops advancing would otherwise spin until the step budget ran out and
|
|
88
|
+
// report nothing useful. Refusing names the store and the count, which is what an operator
|
|
89
|
+
// needs to tell "enormous catalog" from "cursor stuck".
|
|
90
|
+
throw new TaskNonRetryableError(`channel/${label} for store ${storeId} did not exhaust within ${CHANNEL_MAX_BATCHES_PER_SWEEP} batches `
|
|
91
|
+
+ `(${counted} items walked). Either the catalog is larger than this sweep supports or the cursor is not advancing.`);
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
}
|
|
15
95
|
export { signState, verifyState } from "./oauth-state.js";
|
|
16
96
|
function unwrap(result) {
|
|
17
97
|
if (result.ok)
|
|
@@ -88,45 +168,51 @@ export function channelConnectorPlugin(options = {}) {
|
|
|
88
168
|
{
|
|
89
169
|
slug: "channel/import-catalog",
|
|
90
170
|
concurrency: { key: (input) => String(input.storeId), supersedes: true },
|
|
171
|
+
durableSteps: true,
|
|
91
172
|
handler: async ({ input, ctx }) => {
|
|
92
173
|
const service = new ChannelConnectorService(ctx.db, ctx.services, options);
|
|
93
174
|
const orgId = String(input.orgId);
|
|
94
175
|
const storeId = String(input.storeId);
|
|
95
|
-
const result = await
|
|
96
|
-
|
|
176
|
+
const result = await walkBatches(ctx, "import-catalog", storeId, async () => {
|
|
177
|
+
const page = await service.importCatalog(orgId, storeId, createSystemActor(orgId), {
|
|
178
|
+
maxItems: CHANNEL_IMPORT_MAX_ITEMS_PER_INVOCATION,
|
|
179
|
+
});
|
|
180
|
+
if (!page.ok)
|
|
181
|
+
throw new Error(page.error);
|
|
182
|
+
return {
|
|
183
|
+
exhausted: page.value.exhausted,
|
|
184
|
+
counted: page.value.imported,
|
|
185
|
+
cursor: page.value.cursor ?? null,
|
|
186
|
+
...(page.value.warnings ? { warnings: page.value.warnings } : {}),
|
|
187
|
+
};
|
|
97
188
|
});
|
|
98
|
-
if (!result.ok)
|
|
99
|
-
throw new Error(result.error);
|
|
100
189
|
const jobs = ctx.services.jobs;
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
organizationId: orgId,
|
|
121
|
-
concurrencyKey: storeId,
|
|
122
|
-
});
|
|
123
|
-
}
|
|
190
|
+
// The catalog is always exhausted by the time the walk above returns, so this hand-off is
|
|
191
|
+
// unconditional. It is the ONE enqueue this task is allowed: it starts a DIFFERENT task
|
|
192
|
+
// once, spending a single level of the request chain's 32, rather than one per page the
|
|
193
|
+
// way the continuation it replaced did.
|
|
194
|
+
//
|
|
195
|
+
// A finished catalog is not a usable one. `importCatalog` writes entities, variants and
|
|
196
|
+
// prices and never touches `inventory_levels`, so a store whose sweep ends here has a
|
|
197
|
+
// catalog in which every variant rolls up as out of stock — which is what a consumer
|
|
198
|
+
// projection publishes and what a shopper is shown.
|
|
199
|
+
//
|
|
200
|
+
// Inventory used to arrive from `reconcile`, reachable only through the
|
|
201
|
+
// `channel/reconcile-sweep` cron. A deployment that removes its crons therefore loses a
|
|
202
|
+
// data-plane write silently, with every suite still green. Levelling inventory here keeps
|
|
203
|
+
// it inside the one operator action — "import this store" — instead of behind a second
|
|
204
|
+
// one somebody has to remember.
|
|
205
|
+
await jobs.enqueue("channel/sync-inventory", { orgId, storeId }, {
|
|
206
|
+
organizationId: orgId,
|
|
207
|
+
concurrencyKey: storeId,
|
|
208
|
+
});
|
|
124
209
|
return {
|
|
125
210
|
output: {
|
|
126
|
-
imported: result.
|
|
127
|
-
cursor: result.
|
|
128
|
-
exhausted:
|
|
129
|
-
|
|
211
|
+
imported: result.counted,
|
|
212
|
+
cursor: result.last.cursor ?? null,
|
|
213
|
+
exhausted: true,
|
|
214
|
+
batches: result.batches,
|
|
215
|
+
...(result.warnings.length > 0 ? { warnings: result.warnings } : {}),
|
|
130
216
|
},
|
|
131
217
|
};
|
|
132
218
|
},
|
|
@@ -160,23 +246,25 @@ export function channelConnectorPlugin(options = {}) {
|
|
|
160
246
|
{
|
|
161
247
|
slug: "channel/sync-inventory",
|
|
162
248
|
concurrency: { key: (input) => String(input.storeId) },
|
|
249
|
+
durableSteps: true,
|
|
163
250
|
handler: async ({ input, ctx }) => {
|
|
164
251
|
const service = new ChannelConnectorService(ctx.db, ctx.services, options);
|
|
165
252
|
const orgId = String(input.orgId);
|
|
166
253
|
const storeId = String(input.storeId);
|
|
167
|
-
const result = await
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
if (!result.ok)
|
|
171
|
-
throw new Error(result.error);
|
|
172
|
-
const jobs = ctx.services.jobs;
|
|
173
|
-
if (!result.value.exhausted) {
|
|
174
|
-
await jobs.enqueue("channel/sync-inventory", { orgId, storeId }, {
|
|
175
|
-
organizationId: orgId,
|
|
176
|
-
concurrencyKey: storeId,
|
|
254
|
+
const result = await walkBatches(ctx, "sync-inventory", storeId, async () => {
|
|
255
|
+
const batch = await service.syncInventory(orgId, storeId, createSystemActor(orgId), {
|
|
256
|
+
maxItems: CHANNEL_INVENTORY_MAX_ITEMS_PER_INVOCATION,
|
|
177
257
|
});
|
|
178
|
-
|
|
179
|
-
|
|
258
|
+
if (!batch.ok)
|
|
259
|
+
throw new Error(batch.error);
|
|
260
|
+
// `exhausted` is optional on the sync result, and the shape this replaces read a
|
|
261
|
+
// missing one as NOT exhausted (`if (!result.value.exhausted)` chained another batch).
|
|
262
|
+
// Keeping that reading exactly: an absent flag continues, and a cursor that never
|
|
263
|
+
// reports exhaustion is caught loudly by the batch ceiling rather than stopping the
|
|
264
|
+
// sweep early and leaving the store half-levelled.
|
|
265
|
+
return { exhausted: batch.value.exhausted === true, counted: batch.value.synced, cursor: null };
|
|
266
|
+
});
|
|
267
|
+
return { output: { synced: result.counted, exhausted: true, batches: result.batches } };
|
|
180
268
|
},
|
|
181
269
|
},
|
|
182
270
|
{
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@porulle/plugin-channel-connector",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.34.0",
|
|
4
4
|
"license": "MIT",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"exports": {
|
|
@@ -19,15 +19,15 @@
|
|
|
19
19
|
"dependencies": {
|
|
20
20
|
"@hono/zod-openapi": "^1.2.2",
|
|
21
21
|
"hono": "^4.12.5",
|
|
22
|
-
"@porulle/core": "0.
|
|
22
|
+
"@porulle/core": "0.34.0"
|
|
23
23
|
},
|
|
24
24
|
"devDependencies": {
|
|
25
25
|
"@types/node": "^24.5.2",
|
|
26
26
|
"eslint": "^9.39.1",
|
|
27
27
|
"typescript": "5.9.2",
|
|
28
28
|
"vitest": "^3.2.4",
|
|
29
|
-
"@porulle/
|
|
30
|
-
"@porulle/
|
|
29
|
+
"@porulle/typescript-config": "0.1.0",
|
|
30
|
+
"@porulle/eslint-config": "0.1.0"
|
|
31
31
|
},
|
|
32
32
|
"publishConfig": {
|
|
33
33
|
"access": "public"
|
package/src/index.ts
CHANGED
|
@@ -9,6 +9,7 @@ import {
|
|
|
9
9
|
createSystemActor,
|
|
10
10
|
isValidFieldPath,
|
|
11
11
|
requireUserId,
|
|
12
|
+
TaskNonRetryableError,
|
|
12
13
|
} from "@porulle/core";
|
|
13
14
|
import type { FieldPath, JobsAdapter, PluginResult, PluginRouteRegistration, TaskDefinition } from "@porulle/core";
|
|
14
15
|
import { z } from "@hono/zod-openapi";
|
|
@@ -78,6 +79,103 @@ export type {
|
|
|
78
79
|
/** ~320 Neon HTTP subrequests per product against a 10,000 per-invocation cap → hard ceiling near 31; 20 leaves margin for heavier products. */
|
|
79
80
|
export const CHANNEL_IMPORT_MAX_ITEMS_PER_INVOCATION = 20;
|
|
80
81
|
|
|
82
|
+
/**
|
|
83
|
+
* How many bounded batches one sweep may walk before it refuses rather than loops.
|
|
84
|
+
*
|
|
85
|
+
* A batched task walks its whole store inside ONE Workflow instance, so the ceiling that matters is
|
|
86
|
+
* the Workflows steps-per-instance limit: 10,000 on Workers Paid by default, raisable to 25,000
|
|
87
|
+
* (`limits.steps`), and 1,024 on Free. Every batch is one step. A 1,000-product merchant is roughly
|
|
88
|
+
* 650 inventory batches, so this bound is about eight times the largest real sweep and still an
|
|
89
|
+
* order of magnitude under the platform default — it exists to turn a cursor that stops advancing
|
|
90
|
+
* into a loud failure, not to ration normal work.
|
|
91
|
+
*
|
|
92
|
+
* It is NOT the limit that used to kill these sweeps. That was the request-chain depth of 32 Worker
|
|
93
|
+
* invocations, which a self-enqueueing continuation spends one of per batch; see the comment on the
|
|
94
|
+
* sweep loop below.
|
|
95
|
+
*/
|
|
96
|
+
export const CHANNEL_MAX_BATCHES_PER_SWEEP = 5_000;
|
|
97
|
+
|
|
98
|
+
/** What one bounded batch reports back: whether the store is drained, how much work it did, and
|
|
99
|
+
* where it stopped. Everything here crosses a durable-step boundary, so it must stay JSON. */
|
|
100
|
+
interface BatchOutcome {
|
|
101
|
+
exhausted: boolean;
|
|
102
|
+
counted: number;
|
|
103
|
+
cursor: unknown;
|
|
104
|
+
warnings?: string[];
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
/**
|
|
108
|
+
* Walks a store's bounded batches to exhaustion INSIDE THE CALLING INSTANCE, one durable step per
|
|
109
|
+
* batch.
|
|
110
|
+
*
|
|
111
|
+
* WHY THIS SHAPE, in arithmetic rather than in adjectives. Each batch used to create its successor
|
|
112
|
+
* by calling `jobs.enqueue` from inside its own running Workflow instance. Cloudflare caps a single
|
|
113
|
+
* request chain at **32 Worker invocations** ("A single request has a maximum of 32 Worker
|
|
114
|
+
* invocations, and each call to a Service binding counts towards this limit" — Service bindings,
|
|
115
|
+
* Runtime APIs), and a continuation created inside its predecessor spends one, permanently. Two
|
|
116
|
+
* deaths on the deployed Worker on 2026-09-15, same error, same step (`porulle-turn:acquire:0`, the
|
|
117
|
+
* coordinator call, the chain's FIRST step):
|
|
118
|
+
*
|
|
119
|
+
* chain begun inside the catalog import -> died after 18 batches, at offset 360
|
|
120
|
+
* chain begun from a fetch handler -> died after 30 batches, at offset 960
|
|
121
|
+
*
|
|
122
|
+
* 18 + the ~14 the import had already spent ≈ 32; 30 + 2 ≈ 32. What varied was never the volume of
|
|
123
|
+
* work — it was the depth the chain STARTED at, which is why this read as an unreproducible "batch
|
|
124
|
+
* 7 one day, batch 37 the next" for three sessions. gflock-100 needs 65 inventory batches. **No
|
|
125
|
+
* chain survives a catalog of any real size at any batch size, and halving the batch doubles the
|
|
126
|
+
* chain**, which is why the intuitive fix is backwards.
|
|
127
|
+
*
|
|
128
|
+
* A `ctx.step.do` is not an invocation of another Worker. It spends no chain depth, so the walk
|
|
129
|
+
* below is flat however many batches it takes. Do not reintroduce an enqueue of the same slug here:
|
|
130
|
+
* that is the defect, and it looks like a one-line convenience.
|
|
131
|
+
*
|
|
132
|
+
* THE BUDGET PER BATCH GOT BIGGER, NOT SMALLER. A Workflow step's default timeout is 10 minutes
|
|
133
|
+
* (Workflows → Sleeping and retrying: limit 5, 10 s delay, exponential backoff, 10-minute timeout),
|
|
134
|
+
* which is exactly the budget the WHOLE sweep used to have — instance 1428d7e0 died on it with
|
|
135
|
+
* `WorkflowTimeoutError: Execution timed out after 600000ms`, having written 232 of ~1,299 levels.
|
|
136
|
+
* Each batch now gets that budget on its own, and a failed batch retries alone from the cursor its
|
|
137
|
+
* predecessor persisted rather than restarting the store.
|
|
138
|
+
*
|
|
139
|
+
* THE NAME IS LOAD-BEARING. The engine keys a step by its name and replays the cached result for a
|
|
140
|
+
* repeat, so a loop naming every step the same finishes instantly, reports success, and writes one
|
|
141
|
+
* batch. The batch index in the name is the only thing preventing that, and
|
|
142
|
+
* `batched-tasks-do-not-chain.test.ts` asserts the names are distinct.
|
|
143
|
+
*/
|
|
144
|
+
async function walkBatches(
|
|
145
|
+
ctx: import("@porulle/core").TaskContext,
|
|
146
|
+
label: string,
|
|
147
|
+
storeId: string,
|
|
148
|
+
runBatch: () => Promise<BatchOutcome>,
|
|
149
|
+
): Promise<{ counted: number; batches: number; last: BatchOutcome; warnings: string[] }> {
|
|
150
|
+
let counted = 0;
|
|
151
|
+
let batches = 0;
|
|
152
|
+
const warnings: string[] = [];
|
|
153
|
+
let last: BatchOutcome = { exhausted: true, counted: 0, cursor: null };
|
|
154
|
+
// `ctx.step` is absent on engines that have not wired one (pg-boss, Inngest, Trigger). Running
|
|
155
|
+
// the batch inline there is the same walk without durability — which is exactly what the drizzle
|
|
156
|
+
// engine's own pass-through step does — so the plugin keeps working rather than throwing on a
|
|
157
|
+
// property it only needs for resumability.
|
|
158
|
+
const step = ctx.step ?? { do: <T,>(_name: string, fn: () => Promise<T>) => fn() };
|
|
159
|
+
|
|
160
|
+
for (;;) {
|
|
161
|
+
// eslint-disable-next-line no-await-in-loop -- batches are sequential by construction: each one resumes from the cursor the previous one persisted.
|
|
162
|
+
last = await step.do(`${label}:${storeId}:batch:${batches}`, runBatch);
|
|
163
|
+
counted += last.counted;
|
|
164
|
+
if (last.warnings) warnings.push(...last.warnings);
|
|
165
|
+
batches += 1;
|
|
166
|
+
if (last.exhausted) return { counted, batches, last, warnings };
|
|
167
|
+
if (batches >= CHANNEL_MAX_BATCHES_PER_SWEEP) {
|
|
168
|
+
// A cursor that stops advancing would otherwise spin until the step budget ran out and
|
|
169
|
+
// report nothing useful. Refusing names the store and the count, which is what an operator
|
|
170
|
+
// needs to tell "enormous catalog" from "cursor stuck".
|
|
171
|
+
throw new TaskNonRetryableError(
|
|
172
|
+
`channel/${label} for store ${storeId} did not exhaust within ${CHANNEL_MAX_BATCHES_PER_SWEEP} batches `
|
|
173
|
+
+ `(${counted} items walked). Either the catalog is larger than this sweep supports or the cursor is not advancing.`,
|
|
174
|
+
);
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
|
|
81
179
|
export { signState, verifyState } from "./oauth-state.js";
|
|
82
180
|
export type {
|
|
83
181
|
BackfillCatalogOptions,
|
|
@@ -196,23 +294,35 @@ export function channelConnectorPlugin(options: ChannelConnectorPluginOptions =
|
|
|
196
294
|
{
|
|
197
295
|
slug: "channel/import-catalog",
|
|
198
296
|
concurrency: { key: (input: Record<string, unknown>) => String(input.storeId), supersedes: true },
|
|
297
|
+
durableSteps: true,
|
|
199
298
|
handler: async ({ input, ctx }: { input: Record<string, unknown>; ctx: import("@porulle/core").TaskContext }) => {
|
|
200
299
|
const service = new ChannelConnectorService(ctx.db, ctx.services, options);
|
|
201
300
|
const orgId = String(input.orgId);
|
|
202
301
|
const storeId = String(input.storeId);
|
|
203
|
-
const result = await
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
302
|
+
const result = await walkBatches(
|
|
303
|
+
ctx,
|
|
304
|
+
"import-catalog",
|
|
305
|
+
storeId,
|
|
306
|
+
async () => {
|
|
307
|
+
const page = await service.importCatalog(orgId, storeId, createSystemActor(orgId), {
|
|
308
|
+
maxItems: CHANNEL_IMPORT_MAX_ITEMS_PER_INVOCATION,
|
|
309
|
+
});
|
|
310
|
+
if (!page.ok) throw new Error(page.error);
|
|
311
|
+
return {
|
|
312
|
+
exhausted: page.value.exhausted,
|
|
313
|
+
counted: page.value.imported,
|
|
314
|
+
cursor: page.value.cursor ?? null,
|
|
315
|
+
...(page.value.warnings ? { warnings: page.value.warnings } : {}),
|
|
316
|
+
};
|
|
317
|
+
},
|
|
318
|
+
);
|
|
207
319
|
const jobs = ctx.services.jobs as JobsAdapter;
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
} else {
|
|
215
|
-
// A finished catalog is not a usable one. `importCatalog` writes entities, variants and
|
|
320
|
+
// The catalog is always exhausted by the time the walk above returns, so this hand-off is
|
|
321
|
+
// unconditional. It is the ONE enqueue this task is allowed: it starts a DIFFERENT task
|
|
322
|
+
// once, spending a single level of the request chain's 32, rather than one per page the
|
|
323
|
+
// way the continuation it replaced did.
|
|
324
|
+
//
|
|
325
|
+
// A finished catalog is not a usable one. `importCatalog` writes entities, variants and
|
|
216
326
|
// prices and never touches `inventory_levels`, so a store whose sweep ends here has a
|
|
217
327
|
// catalog in which every variant rolls up as out of stock — which is what a consumer
|
|
218
328
|
// projection publishes and what a shopper is shown.
|
|
@@ -224,15 +334,15 @@ export function channelConnectorPlugin(options: ChannelConnectorPluginOptions =
|
|
|
224
334
|
// one somebody has to remember.
|
|
225
335
|
await jobs.enqueue("channel/sync-inventory", { orgId, storeId }, {
|
|
226
336
|
organizationId: orgId,
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
}
|
|
337
|
+
concurrencyKey: storeId,
|
|
338
|
+
});
|
|
230
339
|
return {
|
|
231
340
|
output: {
|
|
232
|
-
imported: result.
|
|
233
|
-
cursor: result.
|
|
234
|
-
exhausted:
|
|
235
|
-
|
|
341
|
+
imported: result.counted,
|
|
342
|
+
cursor: result.last.cursor ?? null,
|
|
343
|
+
exhausted: true,
|
|
344
|
+
batches: result.batches,
|
|
345
|
+
...(result.warnings.length > 0 ? { warnings: result.warnings } : {}),
|
|
236
346
|
},
|
|
237
347
|
};
|
|
238
348
|
},
|
|
@@ -265,22 +375,29 @@ export function channelConnectorPlugin(options: ChannelConnectorPluginOptions =
|
|
|
265
375
|
{
|
|
266
376
|
slug: "channel/sync-inventory",
|
|
267
377
|
concurrency: { key: (input: Record<string, unknown>) => String(input.storeId) },
|
|
378
|
+
durableSteps: true,
|
|
268
379
|
handler: async ({ input, ctx }: { input: Record<string, unknown>; ctx: import("@porulle/core").TaskContext }) => {
|
|
269
380
|
const service = new ChannelConnectorService(ctx.db, ctx.services, options);
|
|
270
381
|
const orgId = String(input.orgId);
|
|
271
382
|
const storeId = String(input.storeId);
|
|
272
|
-
const result = await
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
383
|
+
const result = await walkBatches(
|
|
384
|
+
ctx,
|
|
385
|
+
"sync-inventory",
|
|
386
|
+
storeId,
|
|
387
|
+
async () => {
|
|
388
|
+
const batch = await service.syncInventory(orgId, storeId, createSystemActor(orgId), {
|
|
389
|
+
maxItems: CHANNEL_INVENTORY_MAX_ITEMS_PER_INVOCATION,
|
|
390
|
+
});
|
|
391
|
+
if (!batch.ok) throw new Error(batch.error);
|
|
392
|
+
// `exhausted` is optional on the sync result, and the shape this replaces read a
|
|
393
|
+
// missing one as NOT exhausted (`if (!result.value.exhausted)` chained another batch).
|
|
394
|
+
// Keeping that reading exactly: an absent flag continues, and a cursor that never
|
|
395
|
+
// reports exhaustion is caught loudly by the batch ceiling rather than stopping the
|
|
396
|
+
// sweep early and leaving the store half-levelled.
|
|
397
|
+
return { exhausted: batch.value.exhausted === true, counted: batch.value.synced, cursor: null };
|
|
398
|
+
},
|
|
399
|
+
);
|
|
400
|
+
return { output: { synced: result.counted, exhausted: true, batches: result.batches } };
|
|
284
401
|
},
|
|
285
402
|
},
|
|
286
403
|
{
|