@porulle/plugin-channel-connector 0.28.0 → 0.29.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +2 -0
- package/dist/index.js +16 -1
- package/dist/service.d.ts +11 -1
- package/dist/service.js +111 -17
- package/package.json +4 -4
- package/src/index.ts +17 -1
- package/src/service.ts +152 -16
package/dist/index.d.ts
CHANGED
|
@@ -4,6 +4,8 @@ export type { MockChannelConnectorOptions } from "./mock-connector.js";
|
|
|
4
4
|
export { ChannelConnectorService, CATALOG_OUTBOUND_SUPPRESSION_WINDOW_MS, CATALOG_PUSH_BATCH_SIZES, CATALOG_PUSH_MAX_ATTEMPTS, canCatalogPushTransition, canExportTransition, catalogPushConcurrencyKey, catalogPushRetryDelayMs, isCatalogPushBreakerOpen, } from "./service.js";
|
|
5
5
|
export { isValidCatalogMappingFieldPath, matchFieldPath, mergeCatalogFieldMapping, normalizeCatalogFieldMapping, compareCatalogFieldMappingSpecificity, providerCatalogFieldMappingDefaults, selectCatalogFieldMapping, validateCatalogMappingRow, } from "./catalog-field-mapping.js";
|
|
6
6
|
export type { CatalogFieldMapping, CatalogFieldMappingInput, CatalogFieldMappingRow, CatalogFieldTarget, } from "./catalog-field-mapping.js";
|
|
7
|
+
/** ~320 Neon HTTP subrequests per product against a 10,000 per-invocation cap → hard ceiling near 31; 20 leaves margin for heavier products. */
|
|
8
|
+
export declare const CHANNEL_IMPORT_MAX_ITEMS_PER_INVOCATION = 20;
|
|
7
9
|
export { signState, verifyState } from "./oauth-state.js";
|
|
8
10
|
export type { BackfillCatalogOptions, BackfillCatalogReport, BuildCatalogPushItemsOptions, BuildCatalogPushItemsResult, CatalogPushAssemblyField, CatalogPushAssemblyImage, CatalogPushAssemblyItem, CatalogPushPreviewBefore, CatalogPushPreviewBeforeStatus, CatalogPushPreviewDiff, CatalogPushPreviewItem, CatalogPushPreviewResult, CatalogPushPreviewUnavailable, PushCatalogToStoreResult, CatalogPushJobResult, CatalogFieldConflict, CatalogFieldSkip, CatalogPushFieldSkip, CatalogPushSkipReason, CatalogConflictState, CatalogWriteSettings, ChannelComplianceData, ChannelConnectorPluginOptions, ChannelStockLine, ExportState, PublicConnectedStore, ReconcileReport, } from "./service.js";
|
|
9
11
|
export type { OAuthStatePayload, OAuthStateResult } from "./oauth-state.js";
|
package/dist/index.js
CHANGED
|
@@ -10,6 +10,8 @@ import { oauthStateEventId, signState, verifyState } from "./oauth-state.js";
|
|
|
10
10
|
export { mockChannelConnector } from "./mock-connector.js";
|
|
11
11
|
export { ChannelConnectorService, CATALOG_OUTBOUND_SUPPRESSION_WINDOW_MS, CATALOG_PUSH_BATCH_SIZES, CATALOG_PUSH_MAX_ATTEMPTS, canCatalogPushTransition, canExportTransition, catalogPushConcurrencyKey, catalogPushRetryDelayMs, isCatalogPushBreakerOpen, } from "./service.js";
|
|
12
12
|
export { isValidCatalogMappingFieldPath, matchFieldPath, mergeCatalogFieldMapping, normalizeCatalogFieldMapping, compareCatalogFieldMappingSpecificity, providerCatalogFieldMappingDefaults, selectCatalogFieldMapping, validateCatalogMappingRow, } from "./catalog-field-mapping.js";
|
|
13
|
+
/** ~320 Neon HTTP subrequests per product against a 10,000 per-invocation cap → hard ceiling near 31; 20 leaves margin for heavier products. */
|
|
14
|
+
export const CHANNEL_IMPORT_MAX_ITEMS_PER_INVOCATION = 20;
|
|
13
15
|
export { signState, verifyState } from "./oauth-state.js";
|
|
14
16
|
function unwrap(result) {
|
|
15
17
|
if (result.ok)
|
|
@@ -88,13 +90,26 @@ export function channelConnectorPlugin(options = {}) {
|
|
|
88
90
|
concurrency: { key: (input) => String(input.storeId), supersedes: true },
|
|
89
91
|
handler: async ({ input, ctx }) => {
|
|
90
92
|
const service = new ChannelConnectorService(ctx.db, ctx.services, options);
|
|
91
|
-
const
|
|
93
|
+
const orgId = String(input.orgId);
|
|
94
|
+
const storeId = String(input.storeId);
|
|
95
|
+
const result = await service.importCatalog(orgId, storeId, createSystemActor(orgId), {
|
|
96
|
+
maxItems: CHANNEL_IMPORT_MAX_ITEMS_PER_INVOCATION,
|
|
97
|
+
});
|
|
92
98
|
if (!result.ok)
|
|
93
99
|
throw new Error(result.error);
|
|
100
|
+
if (!result.value.exhausted) {
|
|
101
|
+
const jobs = ctx.services.jobs;
|
|
102
|
+
await jobs.enqueue("channel/import-catalog", { orgId, storeId }, {
|
|
103
|
+
organizationId: orgId,
|
|
104
|
+
concurrencyKey: storeId,
|
|
105
|
+
supersedes: true,
|
|
106
|
+
});
|
|
107
|
+
}
|
|
94
108
|
return {
|
|
95
109
|
output: {
|
|
96
110
|
imported: result.value.imported,
|
|
97
111
|
cursor: result.value.cursor,
|
|
112
|
+
exhausted: result.value.exhausted,
|
|
98
113
|
...(result.value.warnings ? { warnings: result.value.warnings } : {}),
|
|
99
114
|
},
|
|
100
115
|
};
|
package/dist/service.d.ts
CHANGED
|
@@ -216,7 +216,17 @@ export declare class ChannelConnectorService {
|
|
|
216
216
|
getStore(orgId: string, id: string): Promise<PluginResult<PublicConnectedStore>>;
|
|
217
217
|
listStores(orgId: string): Promise<PluginResult<PublicConnectedStore[]>>;
|
|
218
218
|
validateLineStock(orgId: string, lines: ChannelStockLine[], timeoutMs?: number): Promise<void>;
|
|
219
|
-
importCatalog(orgId: string, storeId: string, actor: Actor
|
|
219
|
+
importCatalog(orgId: string, storeId: string, actor: Actor, options: {
|
|
220
|
+
maxItems: number;
|
|
221
|
+
}): Promise<PluginResult<{
|
|
222
|
+
imported: number;
|
|
223
|
+
cursor: string | null;
|
|
224
|
+
exhausted: boolean;
|
|
225
|
+
skipped?: CatalogFieldSkip[];
|
|
226
|
+
conflicts?: CatalogFieldConflict[];
|
|
227
|
+
warnings?: string[];
|
|
228
|
+
}>>;
|
|
229
|
+
importCatalog(orgId: string, storeId: string, actor: Actor, options?: undefined): Promise<PluginResult<{
|
|
220
230
|
imported: number;
|
|
221
231
|
cursor: string | null;
|
|
222
232
|
skipped?: CatalogFieldSkip[];
|
package/dist/service.js
CHANGED
|
@@ -48,6 +48,26 @@ export function isCatalogPushBreakerOpen(breakerState) {
|
|
|
48
48
|
function catalogPushBatchSize(provider) {
|
|
49
49
|
return CATALOG_PUSH_BATCH_SIZES[provider] ?? DEFAULT_CATALOG_PUSH_BATCH_SIZE;
|
|
50
50
|
}
|
|
51
|
+
function parseImportResumePosition(raw) {
|
|
52
|
+
if (!raw)
|
|
53
|
+
return { pageCursor: null, offset: 0 };
|
|
54
|
+
try {
|
|
55
|
+
const parsed = JSON.parse(raw);
|
|
56
|
+
if (typeof parsed === "object" && parsed !== null && "offset" in parsed) {
|
|
57
|
+
return {
|
|
58
|
+
pageCursor: parsed.pageCursor ?? null,
|
|
59
|
+
offset: typeof parsed.offset === "number" && parsed.offset >= 0 ? parsed.offset : 0,
|
|
60
|
+
};
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
catch {
|
|
64
|
+
// Legacy bare page cursor.
|
|
65
|
+
}
|
|
66
|
+
return { pageCursor: raw, offset: 0 };
|
|
67
|
+
}
|
|
68
|
+
function encodeImportResumePosition(position) {
|
|
69
|
+
return JSON.stringify(position);
|
|
70
|
+
}
|
|
51
71
|
const exportTransitions = {
|
|
52
72
|
pending: ["exported", "abandoned"],
|
|
53
73
|
exported: ["confirmed", "failed", "abandoned"],
|
|
@@ -1538,7 +1558,7 @@ export class ChannelConnectorService {
|
|
|
1538
1558
|
}
|
|
1539
1559
|
}));
|
|
1540
1560
|
}
|
|
1541
|
-
async importCatalog(orgId, storeId, actor) {
|
|
1561
|
+
async importCatalog(orgId, storeId, actor, options) {
|
|
1542
1562
|
const store = await this.getStoreRecord(orgId, storeId);
|
|
1543
1563
|
if (!store || store.status !== "connected") {
|
|
1544
1564
|
return PluginErr("Connected store not found.", "NOT_FOUND");
|
|
@@ -1546,28 +1566,97 @@ export class ChannelConnectorService {
|
|
|
1546
1566
|
const connector = this.connectors.get(store.provider);
|
|
1547
1567
|
if (!connector)
|
|
1548
1568
|
return PluginErr(`No connector registered for provider "${store.provider}".`);
|
|
1549
|
-
const
|
|
1550
|
-
|
|
1551
|
-
|
|
1552
|
-
const
|
|
1569
|
+
const maxItems = options?.maxItems;
|
|
1570
|
+
const bounded = maxItems !== undefined;
|
|
1571
|
+
if (!bounded) {
|
|
1572
|
+
const resume = parseImportResumePosition(store.catalogCursor);
|
|
1573
|
+
const items = [];
|
|
1574
|
+
let pageCursor = resume.pageCursor ?? undefined;
|
|
1575
|
+
do {
|
|
1576
|
+
const page = await connector.importCatalog(store, pageCursor);
|
|
1577
|
+
if (!page.ok)
|
|
1578
|
+
return PluginErr(page.error.message);
|
|
1579
|
+
items.push(...page.value.items);
|
|
1580
|
+
pageCursor = page.value.nextCursor ?? undefined;
|
|
1581
|
+
} while (pageCursor);
|
|
1582
|
+
const result = await this.convergeCatalogItems(orgId, storeId, items, actor);
|
|
1583
|
+
if (!result.ok)
|
|
1584
|
+
return result;
|
|
1585
|
+
await this.db
|
|
1586
|
+
.update(connectedStores)
|
|
1587
|
+
.set({ catalogCursor: null, lastSyncAt: new Date(), updatedAt: new Date() })
|
|
1588
|
+
.where(and(eq(connectedStores.organizationId, orgId), eq(connectedStores.id, storeId)));
|
|
1589
|
+
return Ok({
|
|
1590
|
+
imported: result.value.imported,
|
|
1591
|
+
cursor: null,
|
|
1592
|
+
...(result.value.skipped.length > 0 ? { skipped: uniqueSkipped(result.value.skipped) } : {}),
|
|
1593
|
+
...(result.value.conflicts.length > 0 ? { conflicts: result.value.conflicts } : {}),
|
|
1594
|
+
...(result.value.warnings.length > 0 ? { warnings: result.value.warnings } : {}),
|
|
1595
|
+
});
|
|
1596
|
+
}
|
|
1597
|
+
let { pageCursor, offset } = parseImportResumePosition(store.catalogCursor);
|
|
1598
|
+
let remaining = maxItems;
|
|
1599
|
+
let exhausted = false;
|
|
1600
|
+
let totalImported = 0;
|
|
1601
|
+
const skipped = [];
|
|
1602
|
+
const conflicts = [];
|
|
1603
|
+
const warnings = [];
|
|
1604
|
+
while (remaining > 0) {
|
|
1605
|
+
const page = await connector.importCatalog(store, pageCursor ?? undefined);
|
|
1553
1606
|
if (!page.ok)
|
|
1554
1607
|
return PluginErr(page.error.message);
|
|
1555
|
-
|
|
1556
|
-
|
|
1557
|
-
|
|
1558
|
-
|
|
1559
|
-
|
|
1560
|
-
|
|
1608
|
+
const pageItems = page.value.items;
|
|
1609
|
+
const slice = pageItems.slice(offset);
|
|
1610
|
+
const batchSize = Math.min(remaining, slice.length);
|
|
1611
|
+
if (batchSize === 0) {
|
|
1612
|
+
if (page.value.nextCursor) {
|
|
1613
|
+
pageCursor = page.value.nextCursor;
|
|
1614
|
+
offset = 0;
|
|
1615
|
+
continue;
|
|
1616
|
+
}
|
|
1617
|
+
exhausted = true;
|
|
1618
|
+
break;
|
|
1619
|
+
}
|
|
1620
|
+
const batch = slice.slice(0, batchSize);
|
|
1621
|
+
const result = await this.convergeCatalogItems(orgId, storeId, batch, actor);
|
|
1622
|
+
if (!result.ok)
|
|
1623
|
+
return result;
|
|
1624
|
+
totalImported += result.value.imported;
|
|
1625
|
+
remaining -= result.value.consumed;
|
|
1626
|
+
skipped.push(...result.value.skipped);
|
|
1627
|
+
conflicts.push(...result.value.conflicts);
|
|
1628
|
+
warnings.push(...result.value.warnings);
|
|
1629
|
+
offset += result.value.consumed;
|
|
1630
|
+
if (offset >= pageItems.length) {
|
|
1631
|
+
if (page.value.nextCursor) {
|
|
1632
|
+
pageCursor = page.value.nextCursor;
|
|
1633
|
+
offset = 0;
|
|
1634
|
+
}
|
|
1635
|
+
else {
|
|
1636
|
+
exhausted = true;
|
|
1637
|
+
}
|
|
1638
|
+
}
|
|
1639
|
+
if (remaining === 0)
|
|
1640
|
+
break;
|
|
1641
|
+
if (exhausted)
|
|
1642
|
+
break;
|
|
1643
|
+
}
|
|
1644
|
+
const catalogCursor = exhausted ? null : encodeImportResumePosition({ pageCursor, offset });
|
|
1561
1645
|
await this.db
|
|
1562
1646
|
.update(connectedStores)
|
|
1563
|
-
.set({
|
|
1647
|
+
.set({
|
|
1648
|
+
catalogCursor,
|
|
1649
|
+
...(exhausted ? { lastSyncAt: new Date() } : {}),
|
|
1650
|
+
updatedAt: new Date(),
|
|
1651
|
+
})
|
|
1564
1652
|
.where(and(eq(connectedStores.organizationId, orgId), eq(connectedStores.id, storeId)));
|
|
1565
1653
|
return Ok({
|
|
1566
|
-
imported:
|
|
1567
|
-
cursor:
|
|
1568
|
-
|
|
1569
|
-
...(
|
|
1570
|
-
...(
|
|
1654
|
+
imported: totalImported,
|
|
1655
|
+
cursor: catalogCursor,
|
|
1656
|
+
exhausted,
|
|
1657
|
+
...(skipped.length > 0 ? { skipped: uniqueSkipped(skipped) } : {}),
|
|
1658
|
+
...(conflicts.length > 0 ? { conflicts } : {}),
|
|
1659
|
+
...(warnings.length > 0 ? { warnings } : {}),
|
|
1571
1660
|
});
|
|
1572
1661
|
}
|
|
1573
1662
|
async promoteLegacyAttributes(orgId, storeId, actor, dryRun) {
|
|
@@ -1728,12 +1817,14 @@ export class ChannelConnectorService {
|
|
|
1728
1817
|
attributesCreated: 0,
|
|
1729
1818
|
mediaImported: 0,
|
|
1730
1819
|
variantsGivenOptionValues: 0,
|
|
1820
|
+
consumed: 0,
|
|
1731
1821
|
skipped: [],
|
|
1732
1822
|
conflicts: [],
|
|
1733
1823
|
warnings: [],
|
|
1734
1824
|
};
|
|
1735
1825
|
const assets = await this.db.select().from(mediaAssets).where(eq(mediaAssets.organizationId, orgId));
|
|
1736
1826
|
for (const item of items) {
|
|
1827
|
+
stats.consumed += 1;
|
|
1737
1828
|
const [entityMapping] = await this.db.select().from(channelEntityMap).where(and(eq(channelEntityMap.organizationId, orgId), eq(channelEntityMap.storeId, storeId), eq(channelEntityMap.kind, "entity"), eq(channelEntityMap.externalId, item.externalId)));
|
|
1738
1829
|
if (!entityMapping) {
|
|
1739
1830
|
stats.imported += 1;
|
|
@@ -1857,10 +1948,12 @@ export class ChannelConnectorService {
|
|
|
1857
1948
|
let attributesCreated = 0;
|
|
1858
1949
|
let mediaImported = 0;
|
|
1859
1950
|
let variantsGivenOptionValues = 0;
|
|
1951
|
+
let consumed = 0;
|
|
1860
1952
|
const skipped = [];
|
|
1861
1953
|
const conflicts = [];
|
|
1862
1954
|
const warnings = [];
|
|
1863
1955
|
for (const item of items) {
|
|
1956
|
+
consumed += 1;
|
|
1864
1957
|
const remoteHash = hash(item);
|
|
1865
1958
|
const existing = await this.db
|
|
1866
1959
|
.select()
|
|
@@ -2057,6 +2150,7 @@ export class ChannelConnectorService {
|
|
|
2057
2150
|
attributesCreated,
|
|
2058
2151
|
mediaImported,
|
|
2059
2152
|
variantsGivenOptionValues,
|
|
2153
|
+
consumed,
|
|
2060
2154
|
skipped,
|
|
2061
2155
|
conflicts,
|
|
2062
2156
|
warnings,
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@porulle/plugin-channel-connector",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.29.0",
|
|
4
4
|
"license": "MIT",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"exports": {
|
|
@@ -19,15 +19,15 @@
|
|
|
19
19
|
"dependencies": {
|
|
20
20
|
"@hono/zod-openapi": "^1.2.2",
|
|
21
21
|
"hono": "^4.12.5",
|
|
22
|
-
"@porulle/core": "0.
|
|
22
|
+
"@porulle/core": "0.29.0"
|
|
23
23
|
},
|
|
24
24
|
"devDependencies": {
|
|
25
25
|
"@types/node": "^24.5.2",
|
|
26
26
|
"eslint": "^9.39.1",
|
|
27
27
|
"typescript": "5.9.2",
|
|
28
28
|
"vitest": "^3.2.4",
|
|
29
|
-
"@porulle/
|
|
30
|
-
"@porulle/
|
|
29
|
+
"@porulle/eslint-config": "0.1.0",
|
|
30
|
+
"@porulle/typescript-config": "0.1.0"
|
|
31
31
|
},
|
|
32
32
|
"publishConfig": {
|
|
33
33
|
"access": "public"
|
package/src/index.ts
CHANGED
|
@@ -73,6 +73,9 @@ export type {
|
|
|
73
73
|
CatalogFieldMappingRow,
|
|
74
74
|
CatalogFieldTarget,
|
|
75
75
|
} from "./catalog-field-mapping.js";
|
|
76
|
+
/** ~320 Neon HTTP subrequests per product against a 10,000 per-invocation cap → hard ceiling near 31; 20 leaves margin for heavier products. */
|
|
77
|
+
export const CHANNEL_IMPORT_MAX_ITEMS_PER_INVOCATION = 20;
|
|
78
|
+
|
|
76
79
|
export { signState, verifyState } from "./oauth-state.js";
|
|
77
80
|
export type {
|
|
78
81
|
BackfillCatalogOptions,
|
|
@@ -193,12 +196,25 @@ export function channelConnectorPlugin(options: ChannelConnectorPluginOptions =
|
|
|
193
196
|
concurrency: { key: (input: Record<string, unknown>) => String(input.storeId), supersedes: true },
|
|
194
197
|
handler: async ({ input, ctx }: { input: Record<string, unknown>; ctx: import("@porulle/core").TaskContext }) => {
|
|
195
198
|
const service = new ChannelConnectorService(ctx.db, ctx.services, options);
|
|
196
|
-
const
|
|
199
|
+
const orgId = String(input.orgId);
|
|
200
|
+
const storeId = String(input.storeId);
|
|
201
|
+
const result = await service.importCatalog(orgId, storeId, createSystemActor(orgId), {
|
|
202
|
+
maxItems: CHANNEL_IMPORT_MAX_ITEMS_PER_INVOCATION,
|
|
203
|
+
});
|
|
197
204
|
if (!result.ok) throw new Error(result.error);
|
|
205
|
+
if (!result.value.exhausted) {
|
|
206
|
+
const jobs = ctx.services.jobs as JobsAdapter;
|
|
207
|
+
await jobs.enqueue("channel/import-catalog", { orgId, storeId }, {
|
|
208
|
+
organizationId: orgId,
|
|
209
|
+
concurrencyKey: storeId,
|
|
210
|
+
supersedes: true,
|
|
211
|
+
});
|
|
212
|
+
}
|
|
198
213
|
return {
|
|
199
214
|
output: {
|
|
200
215
|
imported: result.value.imported,
|
|
201
216
|
cursor: result.value.cursor,
|
|
217
|
+
exhausted: result.value.exhausted,
|
|
202
218
|
...(result.value.warnings ? { warnings: result.value.warnings } : {}),
|
|
203
219
|
},
|
|
204
220
|
};
|
package/src/service.ts
CHANGED
|
@@ -241,11 +241,34 @@ interface CatalogConvergenceStats {
|
|
|
241
241
|
attributesCreated: number;
|
|
242
242
|
mediaImported: number;
|
|
243
243
|
variantsGivenOptionValues: number;
|
|
244
|
+
consumed: number;
|
|
244
245
|
skipped: CatalogFieldSkip[];
|
|
245
246
|
conflicts: CatalogFieldConflict[];
|
|
246
247
|
warnings: string[];
|
|
247
248
|
}
|
|
248
249
|
|
|
250
|
+
type ImportResumePosition = { pageCursor: string | null; offset: number };
|
|
251
|
+
|
|
252
|
+
function parseImportResumePosition(raw: string | null | undefined): ImportResumePosition {
|
|
253
|
+
if (!raw) return { pageCursor: null, offset: 0 };
|
|
254
|
+
try {
|
|
255
|
+
const parsed = JSON.parse(raw) as Partial<ImportResumePosition>;
|
|
256
|
+
if (typeof parsed === "object" && parsed !== null && "offset" in parsed) {
|
|
257
|
+
return {
|
|
258
|
+
pageCursor: parsed.pageCursor ?? null,
|
|
259
|
+
offset: typeof parsed.offset === "number" && parsed.offset >= 0 ? parsed.offset : 0,
|
|
260
|
+
};
|
|
261
|
+
}
|
|
262
|
+
} catch {
|
|
263
|
+
// Legacy bare page cursor.
|
|
264
|
+
}
|
|
265
|
+
return { pageCursor: raw, offset: 0 };
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
function encodeImportResumePosition(position: ImportResumePosition): string {
|
|
269
|
+
return JSON.stringify(position);
|
|
270
|
+
}
|
|
271
|
+
|
|
249
272
|
export interface BackfillCatalogOptions {
|
|
250
273
|
dryRun?: boolean;
|
|
251
274
|
resume?: boolean;
|
|
@@ -2175,7 +2198,40 @@ export class ChannelConnectorService {
|
|
|
2175
2198
|
orgId: string,
|
|
2176
2199
|
storeId: string,
|
|
2177
2200
|
actor: Actor,
|
|
2178
|
-
|
|
2201
|
+
options: { maxItems: number },
|
|
2202
|
+
): Promise<PluginResult<{
|
|
2203
|
+
imported: number;
|
|
2204
|
+
cursor: string | null;
|
|
2205
|
+
exhausted: boolean;
|
|
2206
|
+
skipped?: CatalogFieldSkip[];
|
|
2207
|
+
conflicts?: CatalogFieldConflict[];
|
|
2208
|
+
warnings?: string[];
|
|
2209
|
+
}>>;
|
|
2210
|
+
async importCatalog(
|
|
2211
|
+
orgId: string,
|
|
2212
|
+
storeId: string,
|
|
2213
|
+
actor: Actor,
|
|
2214
|
+
options?: undefined,
|
|
2215
|
+
): Promise<PluginResult<{
|
|
2216
|
+
imported: number;
|
|
2217
|
+
cursor: string | null;
|
|
2218
|
+
skipped?: CatalogFieldSkip[];
|
|
2219
|
+
conflicts?: CatalogFieldConflict[];
|
|
2220
|
+
warnings?: string[];
|
|
2221
|
+
}>>;
|
|
2222
|
+
async importCatalog(
|
|
2223
|
+
orgId: string,
|
|
2224
|
+
storeId: string,
|
|
2225
|
+
actor: Actor,
|
|
2226
|
+
options?: { maxItems?: number },
|
|
2227
|
+
): Promise<PluginResult<{
|
|
2228
|
+
imported: number;
|
|
2229
|
+
cursor: string | null;
|
|
2230
|
+
exhausted?: boolean;
|
|
2231
|
+
skipped?: CatalogFieldSkip[];
|
|
2232
|
+
conflicts?: CatalogFieldConflict[];
|
|
2233
|
+
warnings?: string[];
|
|
2234
|
+
}>> {
|
|
2179
2235
|
const store = await this.getStoreRecord(orgId, storeId);
|
|
2180
2236
|
if (!store || store.status !== "connected") {
|
|
2181
2237
|
return PluginErr("Connected store not found.", "NOT_FOUND");
|
|
@@ -2183,28 +2239,103 @@ export class ChannelConnectorService {
|
|
|
2183
2239
|
const connector = this.connectors.get(store.provider);
|
|
2184
2240
|
if (!connector) return PluginErr(`No connector registered for provider "${store.provider}".`);
|
|
2185
2241
|
|
|
2186
|
-
const
|
|
2187
|
-
|
|
2188
|
-
|
|
2189
|
-
|
|
2242
|
+
const maxItems = options?.maxItems;
|
|
2243
|
+
const bounded = maxItems !== undefined;
|
|
2244
|
+
|
|
2245
|
+
if (!bounded) {
|
|
2246
|
+
const resume = parseImportResumePosition(store.catalogCursor);
|
|
2247
|
+
const items: ChannelCatalogItem[] = [];
|
|
2248
|
+
let pageCursor: string | undefined = resume.pageCursor ?? undefined;
|
|
2249
|
+
do {
|
|
2250
|
+
const page = await connector.importCatalog(store as ChannelStore, pageCursor);
|
|
2251
|
+
if (!page.ok) return PluginErr(page.error.message);
|
|
2252
|
+
items.push(...page.value.items);
|
|
2253
|
+
pageCursor = page.value.nextCursor ?? undefined;
|
|
2254
|
+
} while (pageCursor);
|
|
2255
|
+
|
|
2256
|
+
const result = await this.convergeCatalogItems(orgId, storeId, items, actor);
|
|
2257
|
+
if (!result.ok) return result;
|
|
2258
|
+
|
|
2259
|
+
await this.db
|
|
2260
|
+
.update(connectedStores)
|
|
2261
|
+
.set({ catalogCursor: null, lastSyncAt: new Date(), updatedAt: new Date() })
|
|
2262
|
+
.where(and(eq(connectedStores.organizationId, orgId), eq(connectedStores.id, storeId)));
|
|
2263
|
+
return Ok({
|
|
2264
|
+
imported: result.value.imported,
|
|
2265
|
+
cursor: null,
|
|
2266
|
+
...(result.value.skipped.length > 0 ? { skipped: uniqueSkipped(result.value.skipped) } : {}),
|
|
2267
|
+
...(result.value.conflicts.length > 0 ? { conflicts: result.value.conflicts } : {}),
|
|
2268
|
+
...(result.value.warnings.length > 0 ? { warnings: result.value.warnings } : {}),
|
|
2269
|
+
});
|
|
2270
|
+
}
|
|
2271
|
+
|
|
2272
|
+
let { pageCursor, offset } = parseImportResumePosition(store.catalogCursor);
|
|
2273
|
+
let remaining = maxItems;
|
|
2274
|
+
let exhausted = false;
|
|
2275
|
+
let totalImported = 0;
|
|
2276
|
+
const skipped: CatalogFieldSkip[] = [];
|
|
2277
|
+
const conflicts: CatalogFieldConflict[] = [];
|
|
2278
|
+
const warnings: string[] = [];
|
|
2279
|
+
|
|
2280
|
+
while (remaining > 0) {
|
|
2281
|
+
const page = await connector.importCatalog(store as ChannelStore, pageCursor ?? undefined);
|
|
2190
2282
|
if (!page.ok) return PluginErr(page.error.message);
|
|
2191
|
-
items.push(...page.value.items);
|
|
2192
|
-
cursor = page.value.nextCursor ?? undefined;
|
|
2193
|
-
} while (cursor);
|
|
2194
2283
|
|
|
2195
|
-
|
|
2196
|
-
|
|
2284
|
+
const pageItems = page.value.items;
|
|
2285
|
+
const slice = pageItems.slice(offset);
|
|
2286
|
+
const batchSize = Math.min(remaining, slice.length);
|
|
2197
2287
|
|
|
2288
|
+
if (batchSize === 0) {
|
|
2289
|
+
if (page.value.nextCursor) {
|
|
2290
|
+
pageCursor = page.value.nextCursor;
|
|
2291
|
+
offset = 0;
|
|
2292
|
+
continue;
|
|
2293
|
+
}
|
|
2294
|
+
exhausted = true;
|
|
2295
|
+
break;
|
|
2296
|
+
}
|
|
2297
|
+
|
|
2298
|
+
const batch = slice.slice(0, batchSize);
|
|
2299
|
+
const result = await this.convergeCatalogItems(orgId, storeId, batch, actor);
|
|
2300
|
+
if (!result.ok) return result;
|
|
2301
|
+
|
|
2302
|
+
totalImported += result.value.imported;
|
|
2303
|
+
remaining -= result.value.consumed;
|
|
2304
|
+
skipped.push(...result.value.skipped);
|
|
2305
|
+
conflicts.push(...result.value.conflicts);
|
|
2306
|
+
warnings.push(...result.value.warnings);
|
|
2307
|
+
offset += result.value.consumed;
|
|
2308
|
+
|
|
2309
|
+
if (offset >= pageItems.length) {
|
|
2310
|
+
if (page.value.nextCursor) {
|
|
2311
|
+
pageCursor = page.value.nextCursor;
|
|
2312
|
+
offset = 0;
|
|
2313
|
+
} else {
|
|
2314
|
+
exhausted = true;
|
|
2315
|
+
}
|
|
2316
|
+
}
|
|
2317
|
+
|
|
2318
|
+
if (remaining === 0) break;
|
|
2319
|
+
if (exhausted) break;
|
|
2320
|
+
}
|
|
2321
|
+
|
|
2322
|
+
const catalogCursor = exhausted ? null : encodeImportResumePosition({ pageCursor, offset });
|
|
2198
2323
|
await this.db
|
|
2199
2324
|
.update(connectedStores)
|
|
2200
|
-
.set({
|
|
2325
|
+
.set({
|
|
2326
|
+
catalogCursor,
|
|
2327
|
+
...(exhausted ? { lastSyncAt: new Date() } : {}),
|
|
2328
|
+
updatedAt: new Date(),
|
|
2329
|
+
})
|
|
2201
2330
|
.where(and(eq(connectedStores.organizationId, orgId), eq(connectedStores.id, storeId)));
|
|
2331
|
+
|
|
2202
2332
|
return Ok({
|
|
2203
|
-
imported:
|
|
2204
|
-
cursor:
|
|
2205
|
-
|
|
2206
|
-
...(
|
|
2207
|
-
...(
|
|
2333
|
+
imported: totalImported,
|
|
2334
|
+
cursor: catalogCursor,
|
|
2335
|
+
exhausted,
|
|
2336
|
+
...(skipped.length > 0 ? { skipped: uniqueSkipped(skipped) } : {}),
|
|
2337
|
+
...(conflicts.length > 0 ? { conflicts } : {}),
|
|
2338
|
+
...(warnings.length > 0 ? { warnings } : {}),
|
|
2208
2339
|
});
|
|
2209
2340
|
}
|
|
2210
2341
|
|
|
@@ -2388,12 +2519,14 @@ export class ChannelConnectorService {
|
|
|
2388
2519
|
attributesCreated: 0,
|
|
2389
2520
|
mediaImported: 0,
|
|
2390
2521
|
variantsGivenOptionValues: 0,
|
|
2522
|
+
consumed: 0,
|
|
2391
2523
|
skipped: [],
|
|
2392
2524
|
conflicts: [],
|
|
2393
2525
|
warnings: [],
|
|
2394
2526
|
};
|
|
2395
2527
|
const assets = await this.db.select().from(mediaAssets).where(eq(mediaAssets.organizationId, orgId));
|
|
2396
2528
|
for (const item of items) {
|
|
2529
|
+
stats.consumed += 1;
|
|
2397
2530
|
const [entityMapping] = await this.db.select().from(channelEntityMap).where(and(
|
|
2398
2531
|
eq(channelEntityMap.organizationId, orgId),
|
|
2399
2532
|
eq(channelEntityMap.storeId, storeId),
|
|
@@ -2533,10 +2666,12 @@ export class ChannelConnectorService {
|
|
|
2533
2666
|
let attributesCreated = 0;
|
|
2534
2667
|
let mediaImported = 0;
|
|
2535
2668
|
let variantsGivenOptionValues = 0;
|
|
2669
|
+
let consumed = 0;
|
|
2536
2670
|
const skipped: CatalogFieldSkip[] = [];
|
|
2537
2671
|
const conflicts: CatalogFieldConflict[] = [];
|
|
2538
2672
|
const warnings: string[] = [];
|
|
2539
2673
|
for (const item of items) {
|
|
2674
|
+
consumed += 1;
|
|
2540
2675
|
const remoteHash = hash(item);
|
|
2541
2676
|
const existing = await this.db
|
|
2542
2677
|
.select()
|
|
@@ -2769,6 +2904,7 @@ export class ChannelConnectorService {
|
|
|
2769
2904
|
attributesCreated,
|
|
2770
2905
|
mediaImported,
|
|
2771
2906
|
variantsGivenOptionValues,
|
|
2907
|
+
consumed,
|
|
2772
2908
|
skipped,
|
|
2773
2909
|
conflicts,
|
|
2774
2910
|
warnings,
|