@porulle/plugin-channel-connector 0.28.0 → 0.29.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.d.ts CHANGED
@@ -4,6 +4,8 @@ export type { MockChannelConnectorOptions } from "./mock-connector.js";
4
4
  export { ChannelConnectorService, CATALOG_OUTBOUND_SUPPRESSION_WINDOW_MS, CATALOG_PUSH_BATCH_SIZES, CATALOG_PUSH_MAX_ATTEMPTS, canCatalogPushTransition, canExportTransition, catalogPushConcurrencyKey, catalogPushRetryDelayMs, isCatalogPushBreakerOpen, } from "./service.js";
5
5
  export { isValidCatalogMappingFieldPath, matchFieldPath, mergeCatalogFieldMapping, normalizeCatalogFieldMapping, compareCatalogFieldMappingSpecificity, providerCatalogFieldMappingDefaults, selectCatalogFieldMapping, validateCatalogMappingRow, } from "./catalog-field-mapping.js";
6
6
  export type { CatalogFieldMapping, CatalogFieldMappingInput, CatalogFieldMappingRow, CatalogFieldTarget, } from "./catalog-field-mapping.js";
7
+ /** ~320 Neon HTTP subrequests per product against a 10,000 per-invocation cap → hard ceiling near 31; 20 leaves margin for heavier products. */
8
+ export declare const CHANNEL_IMPORT_MAX_ITEMS_PER_INVOCATION = 20;
7
9
  export { signState, verifyState } from "./oauth-state.js";
8
10
  export type { BackfillCatalogOptions, BackfillCatalogReport, BuildCatalogPushItemsOptions, BuildCatalogPushItemsResult, CatalogPushAssemblyField, CatalogPushAssemblyImage, CatalogPushAssemblyItem, CatalogPushPreviewBefore, CatalogPushPreviewBeforeStatus, CatalogPushPreviewDiff, CatalogPushPreviewItem, CatalogPushPreviewResult, CatalogPushPreviewUnavailable, PushCatalogToStoreResult, CatalogPushJobResult, CatalogFieldConflict, CatalogFieldSkip, CatalogPushFieldSkip, CatalogPushSkipReason, CatalogConflictState, CatalogWriteSettings, ChannelComplianceData, ChannelConnectorPluginOptions, ChannelStockLine, ExportState, PublicConnectedStore, ReconcileReport, } from "./service.js";
9
11
  export type { OAuthStatePayload, OAuthStateResult } from "./oauth-state.js";
package/dist/index.js CHANGED
@@ -10,6 +10,8 @@ import { oauthStateEventId, signState, verifyState } from "./oauth-state.js";
10
10
  export { mockChannelConnector } from "./mock-connector.js";
11
11
  export { ChannelConnectorService, CATALOG_OUTBOUND_SUPPRESSION_WINDOW_MS, CATALOG_PUSH_BATCH_SIZES, CATALOG_PUSH_MAX_ATTEMPTS, canCatalogPushTransition, canExportTransition, catalogPushConcurrencyKey, catalogPushRetryDelayMs, isCatalogPushBreakerOpen, } from "./service.js";
12
12
  export { isValidCatalogMappingFieldPath, matchFieldPath, mergeCatalogFieldMapping, normalizeCatalogFieldMapping, compareCatalogFieldMappingSpecificity, providerCatalogFieldMappingDefaults, selectCatalogFieldMapping, validateCatalogMappingRow, } from "./catalog-field-mapping.js";
13
+ /** ~320 Neon HTTP subrequests per product against a 10,000 per-invocation cap → hard ceiling near 31; 20 leaves margin for heavier products. */
14
+ export const CHANNEL_IMPORT_MAX_ITEMS_PER_INVOCATION = 20;
13
15
  export { signState, verifyState } from "./oauth-state.js";
14
16
  function unwrap(result) {
15
17
  if (result.ok)
@@ -88,13 +90,26 @@ export function channelConnectorPlugin(options = {}) {
88
90
  concurrency: { key: (input) => String(input.storeId), supersedes: true },
89
91
  handler: async ({ input, ctx }) => {
90
92
  const service = new ChannelConnectorService(ctx.db, ctx.services, options);
91
- const result = await service.importCatalog(String(input.orgId), String(input.storeId), createSystemActor(String(input.orgId)));
93
+ const orgId = String(input.orgId);
94
+ const storeId = String(input.storeId);
95
+ const result = await service.importCatalog(orgId, storeId, createSystemActor(orgId), {
96
+ maxItems: CHANNEL_IMPORT_MAX_ITEMS_PER_INVOCATION,
97
+ });
92
98
  if (!result.ok)
93
99
  throw new Error(result.error);
100
+ if (!result.value.exhausted) {
101
+ const jobs = ctx.services.jobs;
102
+ await jobs.enqueue("channel/import-catalog", { orgId, storeId }, {
103
+ organizationId: orgId,
104
+ concurrencyKey: storeId,
105
+ supersedes: true,
106
+ });
107
+ }
94
108
  return {
95
109
  output: {
96
110
  imported: result.value.imported,
97
111
  cursor: result.value.cursor,
112
+ exhausted: result.value.exhausted,
98
113
  ...(result.value.warnings ? { warnings: result.value.warnings } : {}),
99
114
  },
100
115
  };
package/dist/service.d.ts CHANGED
@@ -216,7 +216,17 @@ export declare class ChannelConnectorService {
216
216
  getStore(orgId: string, id: string): Promise<PluginResult<PublicConnectedStore>>;
217
217
  listStores(orgId: string): Promise<PluginResult<PublicConnectedStore[]>>;
218
218
  validateLineStock(orgId: string, lines: ChannelStockLine[], timeoutMs?: number): Promise<void>;
219
- importCatalog(orgId: string, storeId: string, actor: Actor): Promise<PluginResult<{
219
+ importCatalog(orgId: string, storeId: string, actor: Actor, options: {
220
+ maxItems: number;
221
+ }): Promise<PluginResult<{
222
+ imported: number;
223
+ cursor: string | null;
224
+ exhausted: boolean;
225
+ skipped?: CatalogFieldSkip[];
226
+ conflicts?: CatalogFieldConflict[];
227
+ warnings?: string[];
228
+ }>>;
229
+ importCatalog(orgId: string, storeId: string, actor: Actor, options?: undefined): Promise<PluginResult<{
220
230
  imported: number;
221
231
  cursor: string | null;
222
232
  skipped?: CatalogFieldSkip[];
package/dist/service.js CHANGED
@@ -48,6 +48,26 @@ export function isCatalogPushBreakerOpen(breakerState) {
48
48
  function catalogPushBatchSize(provider) {
49
49
  return CATALOG_PUSH_BATCH_SIZES[provider] ?? DEFAULT_CATALOG_PUSH_BATCH_SIZE;
50
50
  }
51
+ function parseImportResumePosition(raw) {
52
+ if (!raw)
53
+ return { pageCursor: null, offset: 0 };
54
+ try {
55
+ const parsed = JSON.parse(raw);
56
+ if (typeof parsed === "object" && parsed !== null && "offset" in parsed) {
57
+ return {
58
+ pageCursor: parsed.pageCursor ?? null,
59
+ offset: typeof parsed.offset === "number" && parsed.offset >= 0 ? parsed.offset : 0,
60
+ };
61
+ }
62
+ }
63
+ catch {
64
+ // Legacy bare page cursor.
65
+ }
66
+ return { pageCursor: raw, offset: 0 };
67
+ }
68
+ function encodeImportResumePosition(position) {
69
+ return JSON.stringify(position);
70
+ }
51
71
  const exportTransitions = {
52
72
  pending: ["exported", "abandoned"],
53
73
  exported: ["confirmed", "failed", "abandoned"],
@@ -1538,7 +1558,7 @@ export class ChannelConnectorService {
1538
1558
  }
1539
1559
  }));
1540
1560
  }
1541
- async importCatalog(orgId, storeId, actor) {
1561
+ async importCatalog(orgId, storeId, actor, options) {
1542
1562
  const store = await this.getStoreRecord(orgId, storeId);
1543
1563
  if (!store || store.status !== "connected") {
1544
1564
  return PluginErr("Connected store not found.", "NOT_FOUND");
@@ -1546,28 +1566,97 @@ export class ChannelConnectorService {
1546
1566
  const connector = this.connectors.get(store.provider);
1547
1567
  if (!connector)
1548
1568
  return PluginErr(`No connector registered for provider "${store.provider}".`);
1549
- const items = [];
1550
- let cursor = store.catalogCursor ?? undefined;
1551
- do {
1552
- const page = await connector.importCatalog(store, cursor);
1569
+ const maxItems = options?.maxItems;
1570
+ const bounded = maxItems !== undefined;
1571
+ if (!bounded) {
1572
+ const resume = parseImportResumePosition(store.catalogCursor);
1573
+ const items = [];
1574
+ let pageCursor = resume.pageCursor ?? undefined;
1575
+ do {
1576
+ const page = await connector.importCatalog(store, pageCursor);
1577
+ if (!page.ok)
1578
+ return PluginErr(page.error.message);
1579
+ items.push(...page.value.items);
1580
+ pageCursor = page.value.nextCursor ?? undefined;
1581
+ } while (pageCursor);
1582
+ const result = await this.convergeCatalogItems(orgId, storeId, items, actor);
1583
+ if (!result.ok)
1584
+ return result;
1585
+ await this.db
1586
+ .update(connectedStores)
1587
+ .set({ catalogCursor: null, lastSyncAt: new Date(), updatedAt: new Date() })
1588
+ .where(and(eq(connectedStores.organizationId, orgId), eq(connectedStores.id, storeId)));
1589
+ return Ok({
1590
+ imported: result.value.imported,
1591
+ cursor: null,
1592
+ ...(result.value.skipped.length > 0 ? { skipped: uniqueSkipped(result.value.skipped) } : {}),
1593
+ ...(result.value.conflicts.length > 0 ? { conflicts: result.value.conflicts } : {}),
1594
+ ...(result.value.warnings.length > 0 ? { warnings: result.value.warnings } : {}),
1595
+ });
1596
+ }
1597
+ let { pageCursor, offset } = parseImportResumePosition(store.catalogCursor);
1598
+ let remaining = maxItems;
1599
+ let exhausted = false;
1600
+ let totalImported = 0;
1601
+ const skipped = [];
1602
+ const conflicts = [];
1603
+ const warnings = [];
1604
+ while (remaining > 0) {
1605
+ const page = await connector.importCatalog(store, pageCursor ?? undefined);
1553
1606
  if (!page.ok)
1554
1607
  return PluginErr(page.error.message);
1555
- items.push(...page.value.items);
1556
- cursor = page.value.nextCursor ?? undefined;
1557
- } while (cursor);
1558
- const result = await this.convergeCatalogItems(orgId, storeId, items, actor);
1559
- if (!result.ok)
1560
- return result;
1608
+ const pageItems = page.value.items;
1609
+ const slice = pageItems.slice(offset);
1610
+ const batchSize = Math.min(remaining, slice.length);
1611
+ if (batchSize === 0) {
1612
+ if (page.value.nextCursor) {
1613
+ pageCursor = page.value.nextCursor;
1614
+ offset = 0;
1615
+ continue;
1616
+ }
1617
+ exhausted = true;
1618
+ break;
1619
+ }
1620
+ const batch = slice.slice(0, batchSize);
1621
+ const result = await this.convergeCatalogItems(orgId, storeId, batch, actor);
1622
+ if (!result.ok)
1623
+ return result;
1624
+ totalImported += result.value.imported;
1625
+ remaining -= result.value.consumed;
1626
+ skipped.push(...result.value.skipped);
1627
+ conflicts.push(...result.value.conflicts);
1628
+ warnings.push(...result.value.warnings);
1629
+ offset += result.value.consumed;
1630
+ if (offset >= pageItems.length) {
1631
+ if (page.value.nextCursor) {
1632
+ pageCursor = page.value.nextCursor;
1633
+ offset = 0;
1634
+ }
1635
+ else {
1636
+ exhausted = true;
1637
+ }
1638
+ }
1639
+ if (remaining === 0)
1640
+ break;
1641
+ if (exhausted)
1642
+ break;
1643
+ }
1644
+ const catalogCursor = exhausted ? null : encodeImportResumePosition({ pageCursor, offset });
1561
1645
  await this.db
1562
1646
  .update(connectedStores)
1563
- .set({ catalogCursor: null, lastSyncAt: new Date(), updatedAt: new Date() })
1647
+ .set({
1648
+ catalogCursor,
1649
+ ...(exhausted ? { lastSyncAt: new Date() } : {}),
1650
+ updatedAt: new Date(),
1651
+ })
1564
1652
  .where(and(eq(connectedStores.organizationId, orgId), eq(connectedStores.id, storeId)));
1565
1653
  return Ok({
1566
- imported: result.value.imported,
1567
- cursor: null,
1568
- ...(result.value.skipped.length > 0 ? { skipped: uniqueSkipped(result.value.skipped) } : {}),
1569
- ...(result.value.conflicts.length > 0 ? { conflicts: result.value.conflicts } : {}),
1570
- ...(result.value.warnings.length > 0 ? { warnings: result.value.warnings } : {}),
1654
+ imported: totalImported,
1655
+ cursor: catalogCursor,
1656
+ exhausted,
1657
+ ...(skipped.length > 0 ? { skipped: uniqueSkipped(skipped) } : {}),
1658
+ ...(conflicts.length > 0 ? { conflicts } : {}),
1659
+ ...(warnings.length > 0 ? { warnings } : {}),
1571
1660
  });
1572
1661
  }
1573
1662
  async promoteLegacyAttributes(orgId, storeId, actor, dryRun) {
@@ -1728,12 +1817,14 @@ export class ChannelConnectorService {
1728
1817
  attributesCreated: 0,
1729
1818
  mediaImported: 0,
1730
1819
  variantsGivenOptionValues: 0,
1820
+ consumed: 0,
1731
1821
  skipped: [],
1732
1822
  conflicts: [],
1733
1823
  warnings: [],
1734
1824
  };
1735
1825
  const assets = await this.db.select().from(mediaAssets).where(eq(mediaAssets.organizationId, orgId));
1736
1826
  for (const item of items) {
1827
+ stats.consumed += 1;
1737
1828
  const [entityMapping] = await this.db.select().from(channelEntityMap).where(and(eq(channelEntityMap.organizationId, orgId), eq(channelEntityMap.storeId, storeId), eq(channelEntityMap.kind, "entity"), eq(channelEntityMap.externalId, item.externalId)));
1738
1829
  if (!entityMapping) {
1739
1830
  stats.imported += 1;
@@ -1857,10 +1948,12 @@ export class ChannelConnectorService {
1857
1948
  let attributesCreated = 0;
1858
1949
  let mediaImported = 0;
1859
1950
  let variantsGivenOptionValues = 0;
1951
+ let consumed = 0;
1860
1952
  const skipped = [];
1861
1953
  const conflicts = [];
1862
1954
  const warnings = [];
1863
1955
  for (const item of items) {
1956
+ consumed += 1;
1864
1957
  const remoteHash = hash(item);
1865
1958
  const existing = await this.db
1866
1959
  .select()
@@ -2057,6 +2150,7 @@ export class ChannelConnectorService {
2057
2150
  attributesCreated,
2058
2151
  mediaImported,
2059
2152
  variantsGivenOptionValues,
2153
+ consumed,
2060
2154
  skipped,
2061
2155
  conflicts,
2062
2156
  warnings,
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@porulle/plugin-channel-connector",
3
- "version": "0.28.0",
3
+ "version": "0.29.0",
4
4
  "license": "MIT",
5
5
  "type": "module",
6
6
  "exports": {
@@ -19,15 +19,15 @@
19
19
  "dependencies": {
20
20
  "@hono/zod-openapi": "^1.2.2",
21
21
  "hono": "^4.12.5",
22
- "@porulle/core": "0.28.0"
22
+ "@porulle/core": "0.29.0"
23
23
  },
24
24
  "devDependencies": {
25
25
  "@types/node": "^24.5.2",
26
26
  "eslint": "^9.39.1",
27
27
  "typescript": "5.9.2",
28
28
  "vitest": "^3.2.4",
29
- "@porulle/typescript-config": "0.1.0",
30
- "@porulle/eslint-config": "0.1.0"
29
+ "@porulle/eslint-config": "0.1.0",
30
+ "@porulle/typescript-config": "0.1.0"
31
31
  },
32
32
  "publishConfig": {
33
33
  "access": "public"
package/src/index.ts CHANGED
@@ -73,6 +73,9 @@ export type {
73
73
  CatalogFieldMappingRow,
74
74
  CatalogFieldTarget,
75
75
  } from "./catalog-field-mapping.js";
76
+ /** ~320 Neon HTTP subrequests per product against a 10,000 per-invocation cap → hard ceiling near 31; 20 leaves margin for heavier products. */
77
+ export const CHANNEL_IMPORT_MAX_ITEMS_PER_INVOCATION = 20;
78
+
76
79
  export { signState, verifyState } from "./oauth-state.js";
77
80
  export type {
78
81
  BackfillCatalogOptions,
@@ -193,12 +196,25 @@ export function channelConnectorPlugin(options: ChannelConnectorPluginOptions =
193
196
  concurrency: { key: (input: Record<string, unknown>) => String(input.storeId), supersedes: true },
194
197
  handler: async ({ input, ctx }: { input: Record<string, unknown>; ctx: import("@porulle/core").TaskContext }) => {
195
198
  const service = new ChannelConnectorService(ctx.db, ctx.services, options);
196
- const result = await service.importCatalog(String(input.orgId), String(input.storeId), createSystemActor(String(input.orgId)));
199
+ const orgId = String(input.orgId);
200
+ const storeId = String(input.storeId);
201
+ const result = await service.importCatalog(orgId, storeId, createSystemActor(orgId), {
202
+ maxItems: CHANNEL_IMPORT_MAX_ITEMS_PER_INVOCATION,
203
+ });
197
204
  if (!result.ok) throw new Error(result.error);
205
+ if (!result.value.exhausted) {
206
+ const jobs = ctx.services.jobs as JobsAdapter;
207
+ await jobs.enqueue("channel/import-catalog", { orgId, storeId }, {
208
+ organizationId: orgId,
209
+ concurrencyKey: storeId,
210
+ supersedes: true,
211
+ });
212
+ }
198
213
  return {
199
214
  output: {
200
215
  imported: result.value.imported,
201
216
  cursor: result.value.cursor,
217
+ exhausted: result.value.exhausted,
202
218
  ...(result.value.warnings ? { warnings: result.value.warnings } : {}),
203
219
  },
204
220
  };
package/src/service.ts CHANGED
@@ -241,11 +241,34 @@ interface CatalogConvergenceStats {
241
241
  attributesCreated: number;
242
242
  mediaImported: number;
243
243
  variantsGivenOptionValues: number;
244
+ consumed: number;
244
245
  skipped: CatalogFieldSkip[];
245
246
  conflicts: CatalogFieldConflict[];
246
247
  warnings: string[];
247
248
  }
248
249
 
250
+ type ImportResumePosition = { pageCursor: string | null; offset: number };
251
+
252
+ function parseImportResumePosition(raw: string | null | undefined): ImportResumePosition {
253
+ if (!raw) return { pageCursor: null, offset: 0 };
254
+ try {
255
+ const parsed = JSON.parse(raw) as Partial<ImportResumePosition>;
256
+ if (typeof parsed === "object" && parsed !== null && "offset" in parsed) {
257
+ return {
258
+ pageCursor: parsed.pageCursor ?? null,
259
+ offset: typeof parsed.offset === "number" && parsed.offset >= 0 ? parsed.offset : 0,
260
+ };
261
+ }
262
+ } catch {
263
+ // Legacy bare page cursor.
264
+ }
265
+ return { pageCursor: raw, offset: 0 };
266
+ }
267
+
268
+ function encodeImportResumePosition(position: ImportResumePosition): string {
269
+ return JSON.stringify(position);
270
+ }
271
+
249
272
  export interface BackfillCatalogOptions {
250
273
  dryRun?: boolean;
251
274
  resume?: boolean;
@@ -2175,7 +2198,40 @@ export class ChannelConnectorService {
2175
2198
  orgId: string,
2176
2199
  storeId: string,
2177
2200
  actor: Actor,
2178
- ): Promise<PluginResult<{ imported: number; cursor: string | null; skipped?: CatalogFieldSkip[]; conflicts?: CatalogFieldConflict[]; warnings?: string[] }>> {
2201
+ options: { maxItems: number },
2202
+ ): Promise<PluginResult<{
2203
+ imported: number;
2204
+ cursor: string | null;
2205
+ exhausted: boolean;
2206
+ skipped?: CatalogFieldSkip[];
2207
+ conflicts?: CatalogFieldConflict[];
2208
+ warnings?: string[];
2209
+ }>>;
2210
+ async importCatalog(
2211
+ orgId: string,
2212
+ storeId: string,
2213
+ actor: Actor,
2214
+ options?: undefined,
2215
+ ): Promise<PluginResult<{
2216
+ imported: number;
2217
+ cursor: string | null;
2218
+ skipped?: CatalogFieldSkip[];
2219
+ conflicts?: CatalogFieldConflict[];
2220
+ warnings?: string[];
2221
+ }>>;
2222
+ async importCatalog(
2223
+ orgId: string,
2224
+ storeId: string,
2225
+ actor: Actor,
2226
+ options?: { maxItems?: number },
2227
+ ): Promise<PluginResult<{
2228
+ imported: number;
2229
+ cursor: string | null;
2230
+ exhausted?: boolean;
2231
+ skipped?: CatalogFieldSkip[];
2232
+ conflicts?: CatalogFieldConflict[];
2233
+ warnings?: string[];
2234
+ }>> {
2179
2235
  const store = await this.getStoreRecord(orgId, storeId);
2180
2236
  if (!store || store.status !== "connected") {
2181
2237
  return PluginErr("Connected store not found.", "NOT_FOUND");
@@ -2183,28 +2239,103 @@ export class ChannelConnectorService {
2183
2239
  const connector = this.connectors.get(store.provider);
2184
2240
  if (!connector) return PluginErr(`No connector registered for provider "${store.provider}".`);
2185
2241
 
2186
- const items: ChannelCatalogItem[] = [];
2187
- let cursor: string | undefined = store.catalogCursor ?? undefined;
2188
- do {
2189
- const page = await connector.importCatalog(store as ChannelStore, cursor);
2242
+ const maxItems = options?.maxItems;
2243
+ const bounded = maxItems !== undefined;
2244
+
2245
+ if (!bounded) {
2246
+ const resume = parseImportResumePosition(store.catalogCursor);
2247
+ const items: ChannelCatalogItem[] = [];
2248
+ let pageCursor: string | undefined = resume.pageCursor ?? undefined;
2249
+ do {
2250
+ const page = await connector.importCatalog(store as ChannelStore, pageCursor);
2251
+ if (!page.ok) return PluginErr(page.error.message);
2252
+ items.push(...page.value.items);
2253
+ pageCursor = page.value.nextCursor ?? undefined;
2254
+ } while (pageCursor);
2255
+
2256
+ const result = await this.convergeCatalogItems(orgId, storeId, items, actor);
2257
+ if (!result.ok) return result;
2258
+
2259
+ await this.db
2260
+ .update(connectedStores)
2261
+ .set({ catalogCursor: null, lastSyncAt: new Date(), updatedAt: new Date() })
2262
+ .where(and(eq(connectedStores.organizationId, orgId), eq(connectedStores.id, storeId)));
2263
+ return Ok({
2264
+ imported: result.value.imported,
2265
+ cursor: null,
2266
+ ...(result.value.skipped.length > 0 ? { skipped: uniqueSkipped(result.value.skipped) } : {}),
2267
+ ...(result.value.conflicts.length > 0 ? { conflicts: result.value.conflicts } : {}),
2268
+ ...(result.value.warnings.length > 0 ? { warnings: result.value.warnings } : {}),
2269
+ });
2270
+ }
2271
+
2272
+ let { pageCursor, offset } = parseImportResumePosition(store.catalogCursor);
2273
+ let remaining = maxItems;
2274
+ let exhausted = false;
2275
+ let totalImported = 0;
2276
+ const skipped: CatalogFieldSkip[] = [];
2277
+ const conflicts: CatalogFieldConflict[] = [];
2278
+ const warnings: string[] = [];
2279
+
2280
+ while (remaining > 0) {
2281
+ const page = await connector.importCatalog(store as ChannelStore, pageCursor ?? undefined);
2190
2282
  if (!page.ok) return PluginErr(page.error.message);
2191
- items.push(...page.value.items);
2192
- cursor = page.value.nextCursor ?? undefined;
2193
- } while (cursor);
2194
2283
 
2195
- const result = await this.convergeCatalogItems(orgId, storeId, items, actor);
2196
- if (!result.ok) return result;
2284
+ const pageItems = page.value.items;
2285
+ const slice = pageItems.slice(offset);
2286
+ const batchSize = Math.min(remaining, slice.length);
2197
2287
 
2288
+ if (batchSize === 0) {
2289
+ if (page.value.nextCursor) {
2290
+ pageCursor = page.value.nextCursor;
2291
+ offset = 0;
2292
+ continue;
2293
+ }
2294
+ exhausted = true;
2295
+ break;
2296
+ }
2297
+
2298
+ const batch = slice.slice(0, batchSize);
2299
+ const result = await this.convergeCatalogItems(orgId, storeId, batch, actor);
2300
+ if (!result.ok) return result;
2301
+
2302
+ totalImported += result.value.imported;
2303
+ remaining -= result.value.consumed;
2304
+ skipped.push(...result.value.skipped);
2305
+ conflicts.push(...result.value.conflicts);
2306
+ warnings.push(...result.value.warnings);
2307
+ offset += result.value.consumed;
2308
+
2309
+ if (offset >= pageItems.length) {
2310
+ if (page.value.nextCursor) {
2311
+ pageCursor = page.value.nextCursor;
2312
+ offset = 0;
2313
+ } else {
2314
+ exhausted = true;
2315
+ }
2316
+ }
2317
+
2318
+ if (remaining === 0) break;
2319
+ if (exhausted) break;
2320
+ }
2321
+
2322
+ const catalogCursor = exhausted ? null : encodeImportResumePosition({ pageCursor, offset });
2198
2323
  await this.db
2199
2324
  .update(connectedStores)
2200
- .set({ catalogCursor: null, lastSyncAt: new Date(), updatedAt: new Date() })
2325
+ .set({
2326
+ catalogCursor,
2327
+ ...(exhausted ? { lastSyncAt: new Date() } : {}),
2328
+ updatedAt: new Date(),
2329
+ })
2201
2330
  .where(and(eq(connectedStores.organizationId, orgId), eq(connectedStores.id, storeId)));
2331
+
2202
2332
  return Ok({
2203
- imported: result.value.imported,
2204
- cursor: null,
2205
- ...(result.value.skipped.length > 0 ? { skipped: uniqueSkipped(result.value.skipped) } : {}),
2206
- ...(result.value.conflicts.length > 0 ? { conflicts: result.value.conflicts } : {}),
2207
- ...(result.value.warnings.length > 0 ? { warnings: result.value.warnings } : {}),
2333
+ imported: totalImported,
2334
+ cursor: catalogCursor,
2335
+ exhausted,
2336
+ ...(skipped.length > 0 ? { skipped: uniqueSkipped(skipped) } : {}),
2337
+ ...(conflicts.length > 0 ? { conflicts } : {}),
2338
+ ...(warnings.length > 0 ? { warnings } : {}),
2208
2339
  });
2209
2340
  }
2210
2341
 
@@ -2388,12 +2519,14 @@ export class ChannelConnectorService {
2388
2519
  attributesCreated: 0,
2389
2520
  mediaImported: 0,
2390
2521
  variantsGivenOptionValues: 0,
2522
+ consumed: 0,
2391
2523
  skipped: [],
2392
2524
  conflicts: [],
2393
2525
  warnings: [],
2394
2526
  };
2395
2527
  const assets = await this.db.select().from(mediaAssets).where(eq(mediaAssets.organizationId, orgId));
2396
2528
  for (const item of items) {
2529
+ stats.consumed += 1;
2397
2530
  const [entityMapping] = await this.db.select().from(channelEntityMap).where(and(
2398
2531
  eq(channelEntityMap.organizationId, orgId),
2399
2532
  eq(channelEntityMap.storeId, storeId),
@@ -2533,10 +2666,12 @@ export class ChannelConnectorService {
2533
2666
  let attributesCreated = 0;
2534
2667
  let mediaImported = 0;
2535
2668
  let variantsGivenOptionValues = 0;
2669
+ let consumed = 0;
2536
2670
  const skipped: CatalogFieldSkip[] = [];
2537
2671
  const conflicts: CatalogFieldConflict[] = [];
2538
2672
  const warnings: string[] = [];
2539
2673
  for (const item of items) {
2674
+ consumed += 1;
2540
2675
  const remoteHash = hash(item);
2541
2676
  const existing = await this.db
2542
2677
  .select()
@@ -2769,6 +2904,7 @@ export class ChannelConnectorService {
2769
2904
  attributesCreated,
2770
2905
  mediaImported,
2771
2906
  variantsGivenOptionValues,
2907
+ consumed,
2772
2908
  skipped,
2773
2909
  conflicts,
2774
2910
  warnings,