@wix/agent-skills 1.36.0 → 1.36.1

package/skills/wix-replatform/resources/rp-source-shopify/lib/shopify-bulk.js

@@ -1,1470 +0,0 @@
-'use strict';
-
-// The sequential Shopify bulk runner. One bulk operation runs at a time, for both
-// callers: the discovery census and the extract. A caller supplies its own
-// manifest, staging directory, ordered query list and handlers; this module owns every state, stop
-// and recovery rule, so a caller adds no runner and no recovery path of its own.
-//
-// Query states, one atomic manifest write per change:
-//   pending -> starting -> running -> downloaded -> done
-//                              \-> pending (one re-run, or an expired link)
-//   running | downloaded -> settling -> failed | denied      (terminal; the handler runs in between)
-// `settling` makes every terminal side effect recoverable: a query never becomes `failed` or
-// `denied` directly. The runner writes `settling` with the outcome and the verbatim error, runs the
-// outcome's handler, then writes the terminal state with the handler's result. On every start each
-// `settling` entry's handler runs again before anything else, so every handler must be idempotent.
-//
-// run() resolves with { status: 'complete' | 'stopped' | 'awaiting_user_action', code?, message?,
-// details?, manifest }. Anything else it throws is a crash: the manifest on disk is the recovery
-// point, and a fresh runner over the same files resumes it.
-
-const fs = require('node:fs');
-const fsp = require('node:fs/promises');
-const http = require('node:http');
-const https = require('node:https');
-const path = require('node:path');
-const readline = require('node:readline');
-const { fork } = require('node:child_process');
-const { finished } = require('node:stream/promises');
-const { buildShopifyCoverageRows, upsertAdapterCoverageRows, writeJsonAtomic } = require('../../../lib/source-coverage.js');
-const { createUserAction, readUserAction } = require('../../../lib/user-actions.js');
-const { ShopifyCliError } = require('./shopify-cli.js');
-const { ShopifyExtractStop } = require('./import-findings.js');
-const session = require('./shopify-session.js');
-
-const QUERIES_DIR = path.join(__dirname, '..', 'queries');
-const MANIFEST_SCHEMA_VERSION = 1;
-const POLL_START_MS = 2000;
-const POLL_CAP_MS = 30000;
-const OVERDUE_MS = 4 * 60 * 60 * 1000;
-const MAX_ATTEMPTS = 1;
-const MAX_DOWNLOAD_TRIES = 3;
-const MAX_LINK_RERUNS = 2;
-const SNAPSHOT_PLACEHOLDER = '{{SNAPSHOT_TIME}}';
-const OVERDUE_NAMESPACE = 'shopify-bulk-overdue';
-const RAW_EXTRACT_DIR = 'data/source-extract';
-const EXTRACT_MANIFEST_FILE = 'extract-manifest.json';
-const SQLITE_WORKER = path.join(__dirname, 'shopify-bulk-sqlite-worker.js');
-
-const STATUS_QUERY = 'query RpBulkStatus($id: ID!) { node(id: $id) { ... on BulkOperation { id status errorCode createdAt completedAt objectCount rootObjectCount fileSize url partialDataUrl } } }';
-
-// ---------------------------------------------------------------------------------------------
-// The query catalog: eighteen queries, in this order, and how each one's child rows fold
-// ---------------------------------------------------------------------------------------------
-
-const MEDIA_TYPES = ['MediaImage', 'Video', 'ExternalVideo', 'Model3d', 'GenericFile'];
-const DISCOUNT_APPLICATION_TYPES = ['DiscountCodeApplication', 'AutomaticDiscountApplication', 'ManualDiscountApplication', 'ScriptDiscountApplication'];
-
-function typesTo(types, field, extra = {}) {
-  return Object.fromEntries(types.map((type) => [type, { field, ...extra }]));
-}
-
-// `children` maps a child row's `__typename` to the field it folds into on its parent (`into` names
-// an inline object of the parent that holds the field). `rootFields` and `childFields` are the
-// fields initialised to [] so an empty connection reads as empty, not absent. `inline` lists
-// non-connection list fields on a root row whose elements carry nested connections: such a
-// child's `__parentId` may name the inline element, so its objects are indexed by `id`.
-const QUERY_CATALOG = Object.freeze([
-  { name: 'products', entity: 'products', rootFields: ['variants', 'media', 'metafields'], children: { ProductVariant: { field: 'variants' }, ...typesTo(MEDIA_TYPES, 'media'), Metafield: { field: 'metafields' } } },
-  { name: 'collections', entity: 'collections', rootFields: ['products', 'metafields'], children: { Product: { field: 'products' }, Metafield: { field: 'metafields' } } },
-  { name: 'customers', entity: 'customers', rootFields: ['metafields'], children: { Metafield: { field: 'metafields' } } },
-  { name: 'orders-core', entity: 'orders', rootFields: ['lineItems', 'shippingLines', 'discountApplications', 'metafields'], children: { LineItem: { field: 'lineItems' }, ShippingLine: { field: 'shippingLines' }, ...typesTo(DISCOUNT_APPLICATION_TYPES, 'discountApplications'), Metafield: { field: 'metafields' } } },
-  { name: 'orders-refunds', entity: 'orders', inline: { refunds: ['refundLineItems', 'transactions'] }, children: { RefundLineItem: { field: 'refundLineItems' }, OrderTransaction: { field: 'transactions' } } },
-  { name: 'orders-fulfillments', entity: 'orders', inline: { fulfillments: ['fulfillmentLineItems'] }, children: { FulfillmentLineItem: { field: 'fulfillmentLineItems' } } },
-  { name: 'gift-cards', entity: 'gift-cards', children: {} },
-  { name: 'discounts', entity: 'discounts', children: { DiscountRedeemCode: { field: 'codes', into: 'discount' } } },
-  { name: 'delivery-profiles', entity: 'delivery-profiles', rootFields: ['locationGroupZones'], childFields: { DeliveryLocationGroupZone: ['methodDefinitions'] }, children: { DeliveryLocationGroupZone: { field: 'locationGroupZones' }, DeliveryMethodDefinition: { field: 'methodDefinitions' } } },
-  { name: 'inventory', entity: 'inventory-items', rootFields: ['inventoryLevels'], children: { InventoryLevel: { field: 'inventoryLevels' } } },
-  { name: 'pages', entity: 'pages', children: {} },
-  { name: 'blogs', entity: 'blogs', children: {} },
-  { name: 'articles', entity: 'articles', children: {} },
-  { name: 'menus', entity: 'menus', children: {} },
-  { name: 'url-redirects', entity: 'url-redirects', children: {} },
-  { name: 'locations', entity: 'locations', children: {} },
-  { name: 'metaobjects', entity: 'metaobjects', rootFields: ['metaobjects'], children: { Metaobject: { field: 'metaobjects' } } },
-  { name: 'files', entity: 'files', children: {} },
-]);
-
-const QUERY_NAMES = Object.freeze(QUERY_CATALOG.map((entry) => entry.name));
-const CUSTOMER_QUERIES = Object.freeze(['customers']);
-const ORDER_QUERIES = Object.freeze(['orders-core', 'orders-refunds', 'orders-fulfillments']);
-const DENIAL_ATTRIBUTED_QUERIES = Object.freeze([...CUSTOMER_QUERIES, ...ORDER_QUERIES]);
-
-function catalogEntry(name) {
-  const entry = QUERY_CATALOG.find((candidate) => candidate.name === name);
-  if (!entry) throw new Error(`unknown Shopify bulk query ${name}`);
-  return entry;
-}
-
-function queryFilePath(name, dir = QUERIES_DIR) {
-  return path.join(dir, `${name}.graphql`);
-}
-
-function readQueryFile(name, dir = QUERIES_DIR) {
-  return fs.readFileSync(queryFilePath(name, dir), 'utf8');
-}
-
-// The text sent for one query: the file with the snapshot time filled in, then the caller's
-// projection (`projectQuery`), if it passes one, for the customer and order queries.
-function generateQueryText(name, { snapshotTime, project = null, dir = QUERIES_DIR } = {}) {
-  let text = readQueryFile(name, dir);
-  if (text.includes(SNAPSHOT_PLACEHOLDER)) {
-    if (!snapshotTime) throw new Error(`query ${name} needs the snapshot time`);
-    text = text.split(SNAPSHOT_PLACEHOLDER).join(snapshotTime);
-  }
-  if (typeof project === 'function' && DENIAL_ATTRIBUTED_QUERIES.includes(name)) text = project(name, text);
-  return text;
-}
-
-// Static shape checks for the contract test: connections are the `edges { node }` blocks.
-function stripForShape(text) {
-  const noComments = String(text).replace(/#[^\n]*/g, ' ');
-  let out = '';
-  let depth = 0;
-  let inString = false;
-  for (let i = 0; i < noComments.length; i += 1) {
-    const ch = noComments[i];
-    if (inString) {
-      if (ch === '\\') i += 1;
-      else if (ch === '"') inString = false;
-      continue;
-    }
-    if (ch === '"') { inString = true; continue; }
-    if (ch === '(') { depth += 1; continue; }
-    if (ch === ')') { depth = Math.max(0, depth - 1); continue; }
-    if (depth === 0) out += ch;
-  }
-  return out;
-}
-
-function connectionShape(text) {
-  const tokens = stripForShape(text).match(/[{}]|[A-Za-z_][A-Za-z0-9_]*/g) || [];
-  const stack = [];
-  let connections = 0;
-  let maxDepth = 0;
-  let open = 0;
-  for (let i = 0; i < tokens.length; i += 1) {
-    const token = tokens[i];
-    if (token === '{') {
-      const isEdges = tokens[i - 1] === 'edges';
-      stack.push(isEdges);
-      if (isEdges) { open += 1; connections += 1; maxDepth = Math.max(maxDepth, open); }
-    } else if (token === '}') {
-      if (stack.pop()) open -= 1;
-    }
-  }
-  return { connections, nestedLevels: Math.max(0, maxDepth - 1) };
-}
-
-// ---------------------------------------------------------------------------------------------
-// Stops
-// ---------------------------------------------------------------------------------------------
-
-class BulkStop extends Error {
-  constructor(code, message, { status = 'stopped', details = {} } = {}) {
-    super(message);
-    this.name = 'BulkStop';
-    this.code = code;
-    this.status = status;
-    this.details = details;
-  }
-}
-
-const STOPPING_FAILED_CODE = 'shopify_bulk_failed';
-
-function iso(ms) {
-  return new Date(ms).toISOString();
-}
-
-function entityOf(name) {
-  return catalogEntry(name).entity;
-}
-
-function isStopping(entry, caller) {
-  if (caller !== 'extract') return false;
-  if (entry.state === 'failed') return true;
-  return entry.state === 'denied' && entry.handlerOutcome && entry.handlerOutcome.attribution === 'unattributed';
-}
-
-// ---------------------------------------------------------------------------------------------
-// Files: JSONL validation, folding, ndjson writing, the default file host download
-// ---------------------------------------------------------------------------------------------
-
-async function removeFile(file) {
-  await fsp.rm(file, { force: true });
-}
-
-async function writeTextAtomic(file, text) {
-  await fsp.mkdir(path.dirname(file), { recursive: true });
-  const temp = `${file}.${process.pid}.${Date.now()}.tmp`;
-  await fsp.writeFile(temp, text, 'utf8');
-  await fsp.rename(temp, file);
-}
-
-async function writeNdjsonAtomic(file, records) {
-  await fsp.mkdir(path.dirname(file), { recursive: true });
-  const temp = `${file}.${process.pid}.${Date.now()}.tmp`;
-  const out = fs.createWriteStream(temp, { encoding: 'utf8' });
-  let count = 0;
-  try {
-    for await (const record of records) {
-      if (!out.write(`${JSON.stringify(record)}\n`)) await new Promise((resolve) => out.once('drain', resolve));
-      count += 1;
-    }
-    out.end();
-    await finished(out);
-    await fsp.rename(temp, file);
-    return count;
-  } catch (error) {
-    out.destroy();
-    await removeFile(temp);
-    throw error;
-  }
-}
-
-async function* iterateNdjson(file) {
-  const input = fs.createReadStream(file, { encoding: 'utf8' });
-  const lines = readline.createInterface({ input, crlfDelay: Infinity });
-  for await (const line of lines) if (line.length) yield JSON.parse(line);
-}
-
-async function countLines(file) {
-  try {
-    await fsp.access(file);
-  } catch (error) {
-    if (error.code === 'ENOENT') return null;
-    throw error;
-  }
-  let count = 0;
-  for await (const _record of iterateNdjson(file)) count += 1;
-  return count;
-}
-
-async function copyFileAtomic(source, destination) {
-  await fsp.mkdir(path.dirname(destination), { recursive: true });
-  const temp = `${destination}.${process.pid}.${Date.now()}.tmp`;
-  try {
-    await fsp.copyFile(source, temp);
-    await fsp.rename(temp, destination);
-  } catch (error) {
-    await removeFile(temp);
-    throw error;
-  }
-}
-
-// The download gate's file checks: empty or newline-terminated, and every line one
-// JSON object. Streams the file; keeps no record in memory.
-function validateJsonlFile(file) {
-  return new Promise((resolve) => {
-    const stream = fs.createReadStream(file, { encoding: 'utf8' });
-    let rest = '';
-    let lines = 0;
-    let last = '';
-    let failure = null;
-    const checkLine = (line) => {
-      if (failure) return;
-      let value;
-      try {
-        value = JSON.parse(line);
-      } catch {
-        failure = 'a line does not parse as JSON';
-        return;
-      }
-      if (!value || typeof value !== 'object' || Array.isArray(value)) failure = 'a line is not a JSON object';
-      else lines += 1;
-    };
-    stream.on('data', (chunk) => {
-      if (failure) return;
-      last = chunk[chunk.length - 1];
-      rest += chunk;
-      let at = rest.indexOf('\n');
-      while (at !== -1 && !failure) {
-        checkLine(rest.slice(0, at));
-        rest = rest.slice(at + 1);
-        at = rest.indexOf('\n');
-      }
-    });
-    stream.on('error', (error) => resolve({ ok: false, reason: `the file could not be read: ${error.message}` }));
-    stream.on('end', () => {
-      if (!failure && rest.length) failure = 'the file does not end with a newline';
-      if (failure) resolve({ ok: false, reason: failure });
-      else resolve({ ok: true, lines, endsWithNewline: last === '\n' || last === '' });
-    });
-  });
-}
-
-// Node 22.12 ships node:sqlite behind --experimental-sqlite. Keep the documented adapter
-// invocation flag-free: on that one supported boundary runtime, only the disk-backed transform
-// runs in a child with the built-in module enabled. Node 22.13+ stays in-process. The worker gets
-// only paths and the public query shape, never credentials or record values over the command line.
-function loadDatabaseSync() {
-  try {
-    return require('node:sqlite').DatabaseSync;
-  } catch (error) {
-    if (error && error.code === 'ERR_UNKNOWN_BUILTIN_MODULE') return null;
-    throw error;
-  }
-}
-
-function sqliteWorkerError(payload) {
-  const error = new Error(payload && payload.message ? payload.message : 'the SQLite worker failed');
-  if (payload && payload.name) error.name = payload.name;
-  for (const field of ['code', 'errno', 'syscall', 'path']) {
-    if (payload && payload[field] !== undefined) error[field] = payload[field];
-  }
-  if (payload && payload.stack) error.stack = payload.stack;
-  return error;
-}
-
-function runSqliteWorker(operation, payload) {
-  return new Promise((resolve, reject) => {
-    // Preserve explicit memory ceilings used to prove the bounded-memory contract. Other Node
-    // entry-point/debug flags belong to the parent, not to this internal helper.
-    const memoryArgs = process.execArgv.filter((arg) => /^--(?:max-old-space-size|max-semi-space-size|stack-size)=/.test(arg));
-    const child = fork(SQLITE_WORKER, [], {
-      execArgv: [...memoryArgs, '--no-warnings', '--experimental-sqlite'],
-      stdio: ['ignore', 'inherit', 'inherit', 'ipc'],
-    });
-    let settled = false;
-    const fail = (error) => {
-      if (settled) return;
-      settled = true;
-      reject(error);
-    };
-    child.once('error', fail);
-    child.once('exit', (code, signal) => {
-      if (!settled) fail(new Error(`the SQLite worker exited before replying (${signal || `code ${code}`})`));
-    });
-    child.once('message', (message) => {
-      if (settled) return;
-      settled = true;
-      if (child.connected) child.disconnect();
-      if (message && message.ok) resolve(message.result);
-      else reject(sqliteWorkerError(message && message.error));
-    });
-    child.send({ operation, payload }, (error) => {
-      if (error) fail(error);
-    });
-  });
-}
-
-// Shopify leaves `groupObjects` false, so children may precede their parents. Index child rows in
-// a temporary SQLite database, then stream roots in source order and attach only that root's tree.
-// Memory is therefore bounded by the largest assembled root, not by the full store export.
-async function foldBulkJsonl(file, shape, { outputFile } = {}) {
-  if (!outputFile) throw new Error('foldBulkJsonl requires outputFile');
-  const DatabaseSync = loadDatabaseSync();
-  if (!DatabaseSync) return runSqliteWorker('foldBulkJsonl', { file, shape, outputFile });
-  await fsp.mkdir(path.dirname(outputFile), { recursive: true });
-  const work = `${outputFile}.${process.pid}.${Date.now()}.fold.tmp`;
-  const outputTemp = `${outputFile}.${process.pid}.${Date.now()}.tmp`;
-  const db = new DatabaseSync(work);
-  let output = null;
-  let lines = 0;
-  try {
-    db.exec(`
-      PRAGMA journal_mode = OFF;
-      PRAGMA synchronous = OFF;
-      PRAGMA temp_store = FILE;
-      CREATE TABLE roots (seq INTEGER PRIMARY KEY, json TEXT NOT NULL);
-      CREATE TABLE children (
-        seq INTEGER PRIMARY KEY,
-        parent_id TEXT NOT NULL,
-        child_id TEXT,
-        child_type TEXT,
-        json TEXT NOT NULL,
-        attached INTEGER NOT NULL DEFAULT 0,
-        orphan_reason TEXT
-      );
-    `);
-    const addRoot = db.prepare('INSERT INTO roots(seq, json) VALUES (?, ?)');
-    const addChild = db.prepare('INSERT INTO children(seq, parent_id, child_id, child_type, json) VALUES (?, ?, ?, ?, ?)');
-    db.exec('BEGIN');
-    let inTransaction = true;
-    try {
-      for await (const line of readline.createInterface({ input: fs.createReadStream(file, { encoding: 'utf8' }), crlfDelay: Infinity })) {
-        if (!line.length) continue;
-        const row = JSON.parse(line);
-        lines += 1;
-        if (row.__parentId === undefined) addRoot.run(lines, line);
-        else addChild.run(lines, String(row.__parentId), typeof row.id === 'string' ? row.id : null, typeof row.__typename === 'string' ? row.__typename : null, line);
-      }
-      db.exec('COMMIT');
-      inTransaction = false;
-      db.exec('CREATE INDEX children_parent ON children(parent_id, seq)');
-    } catch (error) {
-      if (inTransaction) db.exec('ROLLBACK');
-      throw error;
-    }
-
-    const childrenFor = db.prepare('SELECT seq, json, child_type FROM children WHERE parent_id = ? AND attached = 0 ORDER BY seq');
-    const markAttached = db.prepare('UPDATE children SET attached = 1 WHERE seq = ?');
-    const markOrphan = db.prepare('UPDATE children SET orphan_reason = ? WHERE seq = ?');
-    // Node 22.12's flag-gated StatementSync has no iterate(). Walk the integer primary key with
-    // get() instead: one root is resident at a time and later runtimes follow the identical path.
-    const nextRoot = db.prepare('SELECT seq, json FROM roots WHERE seq > ? ORDER BY seq LIMIT 1');
-    const nestedParentTypes = new Set(Object.keys(shape.childFields || {}));
-    output = fs.createWriteStream(outputTemp, { encoding: 'utf8' });
-    let recordCount = 0;
-    let rootSeq = 0;
-
-    for (let stored = nextRoot.get(rootSeq); stored; stored = nextRoot.get(rootSeq)) {
-      rootSeq = Number(stored.seq);
-      const root = JSON.parse(stored.json);
-      const seen = new Set();
-      for (const field of shape.rootFields || []) if (!Array.isArray(root[field])) root[field] = [];
-      for (const [field, nested] of Object.entries(shape.inline || {})) {
-        for (const element of Array.isArray(root[field]) ? root[field] : []) {
-          if (element && typeof element === 'object') for (const nestedField of nested) if (!Array.isArray(element[nestedField])) element[nestedField] = [];
-        }
-      }
-
-      const attachChildren = (value, mayHaveChildren) => {
-        if (!value || typeof value !== 'object') return;
-        if (mayHaveChildren && typeof value.id === 'string' && !seen.has(value.id)) {
-          seen.add(value.id);
-          for (const storedChild of childrenFor.all(value.id)) {
-            const rule = shape.children[storedChild.child_type];
-            if (!rule) { markOrphan.run('unrecognised_child_type', storedChild.seq); continue; }
-            const holder = rule.into ? (value[rule.into] && typeof value[rule.into] === 'object' ? value[rule.into] : null) : value;
-            if (!holder) { markOrphan.run('parent_not_found', storedChild.seq); continue; }
-            const child = JSON.parse(storedChild.json);
-            delete child.__parentId;
-            if (!Array.isArray(holder[rule.field])) holder[rule.field] = [];
-            holder[rule.field].push(child);
-            markAttached.run(storedChild.seq);
-            attachChildren(child, nestedParentTypes.has(child.__typename));
-          }
-        }
-      };
-      attachChildren(root, true);
-      for (const field of Object.keys(shape.inline || {})) {
-        for (const element of Array.isArray(root[field]) ? root[field] : []) attachChildren(element, true);
-      }
-
-      const initialiseTree = (value) => {
-        if (Array.isArray(value)) { for (const item of value) initialiseTree(item); return; }
-        if (!value || typeof value !== 'object') return;
-        const initialise = shape.childFields && shape.childFields[value.__typename];
-        if (initialise) for (const field of initialise) if (!Array.isArray(value[field])) value[field] = [];
-        for (const nested of Object.values(value)) initialiseTree(nested);
-      };
-      initialiseTree(root);
-      if (!output.write(`${JSON.stringify(root)}\n`)) await new Promise((resolve) => output.once('drain', resolve));
-      recordCount += 1;
-    }
-    output.end();
-    await finished(output);
-    output = null;
-
-    const orphanCount = Number(db.prepare('SELECT COUNT(*) AS count FROM children WHERE attached = 0').get().count);
-    if (orphanCount) {
-      const orphans = db.prepare('SELECT parent_id, child_id, child_type, orphan_reason FROM children WHERE attached = 0 ORDER BY seq LIMIT 20').all().map((row) => ({
-        reason: row.orphan_reason || 'parent_not_found',
-        parentId: row.parent_id,
-        id: row.child_id || null,
-        type: row.child_type || null,
-      }));
-      await removeFile(outputTemp);
-      return { orphans, orphanCount, lines };
-    }
-    await fsp.rename(outputTemp, outputFile);
-    return { outputFile, recordCount, lines };
-  } finally {
-    if (output) output.destroy();
-    db.close();
-    await removeFile(outputTemp);
-    await removeFile(work);
-  }
-}
-
-// Join the three order datasets without loading any of them as a whole. Supplemental rows live
-// in a temporary SQLite table; core orders stream through one at a time in their original order.
-async function joinOrderFiles({ coreFile, refundsFile, fulfillmentsFile, outputFile, workDir = path.dirname(outputFile) }) {
-  const DatabaseSync = loadDatabaseSync();
-  if (!DatabaseSync) return runSqliteWorker('joinOrderFiles', { coreFile, refundsFile, fulfillmentsFile, outputFile, workDir });
-  const work = path.join(workDir, `${path.basename(outputFile)}.${process.pid}.${Date.now()}.orders.tmp`);
-  const outputTemp = path.join(workDir, `${path.basename(outputFile)}.${process.pid}.${Date.now()}.tmp`);
-  const db = new DatabaseSync(work);
-  let output = null;
-  try {
-    db.exec(`
-      PRAGMA journal_mode = OFF;
-      PRAGMA synchronous = OFF;
-      PRAGMA temp_store = FILE;
-      CREATE TABLE supplements (
-        query_name TEXT NOT NULL,
-        order_id TEXT NOT NULL,
-        value TEXT NOT NULL,
-        used INTEGER NOT NULL DEFAULT 0,
-        PRIMARY KEY(query_name, order_id)
-      );
-      CREATE INDEX supplements_order ON supplements(order_id);
-    `);
-    const add = db.prepare('INSERT OR REPLACE INTO supplements(query_name, order_id, value) VALUES (?, ?, ?)');
-    db.exec('BEGIN');
-    try {
-      for (const [query, field, file] of [['orders-refunds', 'refunds', refundsFile], ['orders-fulfillments', 'fulfillments', fulfillmentsFile]]) {
-        if (!file) continue; // this run does not carry that query
-        // A `done` supplement always has its staged file, empty when the dataset is. A missing one
-        // is lost input, never an empty dataset: fail loudly rather than write empty arrays.
-        try {
-          await fsp.access(file);
-        } catch (error) {
-          if (error.code === 'ENOENT') throw new Error(`the staged ${query} file is missing: ${path.basename(file)}`);
-          throw error;
-        }
-        for await (const row of iterateNdjson(file)) add.run(query, String(row.id), JSON.stringify(Array.isArray(row[field]) ? row[field] : []));
-      }
-      db.exec('COMMIT');
-    } catch (error) {
-      db.exec('ROLLBACK');
-      throw error;
-    }
-    const find = db.prepare('SELECT query_name, value FROM supplements WHERE order_id = ?');
-    const markUsed = db.prepare('UPDATE supplements SET used = 1 WHERE order_id = ?');
-    output = fs.createWriteStream(outputTemp, { encoding: 'utf8' });
-    for await (const order of iterateNdjson(coreFile)) {
-      const joined = { ...order, refunds: [], fulfillments: [] };
-      for (const row of find.all(String(order.id))) joined[row.query_name === 'orders-refunds' ? 'refunds' : 'fulfillments'] = JSON.parse(row.value);
-      markUsed.run(String(order.id));
-      if (!output.write(`${JSON.stringify(joined)}\n`)) await new Promise((resolve) => output.once('drain', resolve));
-    }
-    output.end();
-    await finished(output);
-    output = null;
-    await fsp.rename(outputTemp, outputFile);
-    const droppedByQuery = Object.fromEntries(['orders-refunds', 'orders-fulfillments'].map((query) => [query, Number(db.prepare('SELECT COUNT(*) AS count FROM supplements WHERE query_name = ? AND used = 0').get(query).count)]));
-    return { droppedOrphans: droppedByQuery['orders-refunds'] + droppedByQuery['orders-fulfillments'], droppedByQuery };
-  } finally {
-    if (output) output.destroy();
-    db.close();
-    await removeFile(outputTemp);
-    await removeFile(work);
-  }
-}
-
-// The default file host download: plain http(s), no content decoding (identity only), bytes
-// counted as they arrive. It reports what happened; the gate decides what it means.
-function defaultDownload(url, destination) {
-  return new Promise((resolve) => {
-    let client;
-    try {
-      client = String(url).startsWith('http:') ? http : https;
-    } catch {
-      client = https;
-    }
-    const request = client.get(url, { headers: { 'accept-encoding': 'identity' } }, (response) => {
-      if (response.statusCode !== 200) {
-        response.resume();
-        response.on('end', () => resolve({ status: response.statusCode, bytes: 0 }));
-        response.on('error', (error) => resolve({ status: response.statusCode, bytes: 0, error: error.message }));
-        return;
-      }
-      const out = fs.createWriteStream(destination);
-      let bytes = 0;
-      let failed = null;
-      response.on('data', (chunk) => { bytes += chunk.length; });
-      response.on('error', (error) => { failed = error.message; out.end(); });
-      response.on('aborted', () => { failed = failed || 'the connection was cut'; });
-      response.pipe(out);
-      out.on('finish', () => resolve({ status: 200, bytes, ...(failed || !response.complete ? { error: failed || 'the response did not complete' } : {}) }));
-      out.on('error', (error) => resolve({ status: 200, bytes, error: error.message }));
-    });
-    request.on('error', (error) => resolve({ status: null, bytes: 0, error: error.message }));
-  });
-}
-
-// ---------------------------------------------------------------------------------------------
-// The runner
-// ---------------------------------------------------------------------------------------------
-
-function newManifest({ caller, store, snapshotTime, queries, previousSidecars, now }) {
-  const manifest = {
-    schemaVersion: MANIFEST_SCHEMA_VERSION,
-    caller,
-    store,
-    snapshotTime,
-    startedAt: iso(now()),
-    status: 'running',
-    order: queries.map((query) => query.name),
-    queries: Object.fromEntries(queries.map((query) => [query.name, {
-      state: 'pending',
-      entity: query.entity || entityOf(query.name),
-      text: query.text,
-      attempts: 0,
-      reapproved: false,
-      downloadTries: 0,
-      linkReruns: 0,
-    }])),
-  };
-  if (previousSidecars.length) manifest.previousSidecars = [...new Set(previousSidecars)].sort();
-  return manifest;
-}
-
-const OPERATION_FIELDS = ['operationId', 'runningAt', 'deadlineAt', 'overdueRound', 'overdueActionId', 'url', 'fileSize', 'objectCount', 'rootObjectCount', 'createdAt', 'completedAt', 'startedAt'];
-
-// The shopify extract adapters plug into the runner here and add no runner logic of their own: a denial
-// handler, a `failed` handler per query (`failed.byQuery`), a reassembly hook per query, the
-// `denied` reset, and `project` (the projection applied to the customer and order query text).
-// Everything a handler registers must be idempotent. Options passed to createBulkRunner
-// override what is registered.
-const registered = { handlers: {}, deniedReset: null, project: null };
-
-function registerHandlers({ denied, failed, reassemble, deniedReset, project } = {}) {
-  if (denied) registered.handlers.denied = denied;
-  if (failed) registered.handlers.failed = { ...(registered.handlers.failed || {}), ...failed, byQuery: { ...((registered.handlers.failed || {}).byQuery || {}), ...(failed.byQuery || {}) } };
-  if (reassemble) registered.handlers.reassemble = { ...(registered.handlers.reassemble || {}), ...reassemble };
-  if (deniedReset) registered.deniedReset = deniedReset;
-  if (project) registered.project = project;
-}
-
-function clearRegisteredHandlers() {
-  registered.handlers = {};
-  registered.deniedReset = null;
-  registered.project = null;
-}
-
-function mergeHandlers(extra = {}) {
-  const base = registered.handlers;
-  return {
-    denied: extra.denied || base.denied,
-    failed: { ...(base.failed || {}), ...(extra.failed || {}), byQuery: { ...((base.failed || {}).byQuery || {}), ...((extra.failed || {}).byQuery || {}) } },
-    reassemble: { ...(base.reassemble || {}), ...(extra.reassemble || {}) },
-  };
-}
-
-function createBulkRunner(options = {}) {
-  const {
-    projectDir,
-    store,
-    caller,
-    manifestPath,
-    stagingDir,
-    queries = null,
-    snapshotTime = null,
-    execute,
-    handlers: handlerOptions = {},
-    deniedReset = registered.deniedReset,
-    retryFailed = false,
-    previousSidecars = [],
-    sessionCheck = null,
-    download = defaultDownload,
-    outputDir = null,
-    now = () => Date.now(),
-    sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)),
-    progress = () => {},
-    onPersist = null,
-    overdueMs = OVERDUE_MS,
-  } = options;
-  if (caller !== 'extract' && caller !== 'census') throw new Error('caller must be extract or census');
-  if (!projectDir || !manifestPath || !stagingDir) throw new Error('projectDir, manifestPath and stagingDir are required');
-  if (typeof execute !== 'function' || typeof execute.startBulkOperation !== 'function') throw new Error('execute (a store executor with startBulkOperation) is required');
-  if (caller === 'extract' && !outputDir) throw new Error('an extract needs outputDir');
-  const stage = caller === 'extract' ? 'execution' : 'discovery';
-  const extract = caller === 'extract';
-  const normalizedStore = session.normalizeDomain(store);
-  const handlers = mergeHandlers(handlerOptions);
-  let manifest = null;
-
-  // -------------------------------------------------------------------------------------------
-  // Manifest
-  // -------------------------------------------------------------------------------------------
-
-  async function save(label, query = null) {
-    await writeJsonAtomic(manifestPath, manifest);
-    if (typeof onPersist === 'function') await onPersist({ label, query, state: query ? manifest.queries[query].state : null, manifest });
-  }
-
-  async function readManifest() {
-    try {
-      return JSON.parse(await fsp.readFile(manifestPath, 'utf8'));
-    } catch (error) {
-      if (error.code === 'ENOENT') return null;
-      throw error;
-    }
-  }
-
-  const entry = (name) => manifest.queries[name];
-
-  async function setState(name, state, fields = {}, dropFields = []) {
-    const target = entry(name);
-    target.state = state;
-    Object.assign(target, fields);
-    for (const field of dropFields) delete target[field];
-    await save(`${name}:${state}`, name);
-  }
-
-  function dropOperation(name) {
-    const target = entry(name);
-    for (const field of OPERATION_FIELDS) delete target[field];
-  }
-
-  // -------------------------------------------------------------------------------------------
-  // Handlers
-  // -------------------------------------------------------------------------------------------
-
-  function coverageRowFor(name, { artifact, pointer, recoveryAction }) {
-    const entity = entityOf(name);
-    const [row] = buildShopifyCoverageRows({
-      captureFailures: [{ entity, artifact, reads: [name], errors: [{ code: 'shopify_bulk' }] }],
-      rawDir: RAW_EXTRACT_DIR,
-    });
-    return { ...row, evidence: { artifact, pointer }, recoveryAction };
-  }
-
-  function failureEvidence(name, target) {
-    const parts = [`queries.${name}`, `state ${target.outcome || target.state}`, `errorCode ${target.errorCode || 'unknown'}`];
-    if (target.error && target.error.message) parts.push(String(target.error.message).slice(0, 300));
-    return parts.join('; ');
-  }
-
-  // The default `failed` handler. Extract: the capture-failed row, upserted by coverageId.
-  // Census: the same row, carried in the entry's handlerOutcome for the discovery coverage step
-  // (the raw coverage file does not exist until discovery's last step, so nothing is upserted).
-  async function defaultFailedHandler({ name, entry: target }) {
-    const recoveryAction = extract
-      ? `Fix the cause named in the stop message, then run shopify-extract.js --retry-failed for ${entityOf(name)}.`
-      : `Fix the cause, then run Shopify discovery again for ${entityOf(name)} (the extract's gated read also covers it).`;
-    const row = coverageRowFor(name, {
-      artifact: extract ? `${RAW_EXTRACT_DIR}/${EXTRACT_MANIFEST_FILE}` : path.relative(projectDir, manifestPath).split(path.sep).join('/'),
-      pointer: failureEvidence(name, target),
-      recoveryAction,
-    });
-    if (extract) await upsertAdapterCoverageRows(projectDir, { adapter: 'shopify', rows: [row] });
-    return { handlerOutcome: { coverageId: row.coverageId, ...(extract ? {} : { row }) } };
-  }
-
-  // The fail-closed denial fallback, used only when the caller registers no denial handler (the
-  // customer-and-order adapter registers its own): the denial is unattributed, so the dataset is treated like a failed one.
-  async function defaultDeniedHandler(context) {
-    const result = await defaultFailedHandler(context);
-    return { handlerOutcome: { attribution: 'unattributed', ...result.handlerOutcome }, stop: { code: STOPPING_FAILED_CODE, message: `Shopify denied the ${context.name} export and the denial could not be attributed to a withheld detail.` } };
-  }
-
-  function failedHandlerFor(name) {
-    const byQuery = (handlers.failed && handlers.failed.byQuery) || {};
-    return byQuery[name] || (handlers.failed && handlers.failed.default) || defaultFailedHandler;
-  }
-
-  function handlerContext(name) {
-    return { name, query: name, entity: entityOf(name), entry: entry(name), manifest, caller, stage, projectDir, store, execute, stagingDir, outputDir, manifestPath, now, sleep };
-  }
-
-  // -------------------------------------------------------------------------------------------
-  // Settling
-  // -------------------------------------------------------------------------------------------
-
-  async function beginSettling(name, outcome, { errorCode, error }) {
-    await setState(name, 'settling', { outcome, errorCode, error, settlingAt: iso(now()) });
-  }
-
-  // `later` is a start that finds an already-terminal stopping entry: it stops with the generic
-  // `shopify_bulk_failed`; the first settle stops with the entry's own code.
-  function stopFor(name, target, handlerStop, { later = false } = {}) {
-    const entity = entityOf(name);
-    if (handlerStop) {
-      return new BulkStop(handlerStop.code, handlerStop.message || `Shopify ${caller} stopped (${handlerStop.code}) at ${name}.`, { details: { query: name, entity, ...(handlerStop.details || {}) } });
-    }
-    const specific = target.errorCode && target.state === 'failed' ? target.errorCode : STOPPING_FAILED_CODE;
-    const code = later ? STOPPING_FAILED_CODE : specific;
-    const cause = target.error && target.error.message ? `: ${String(target.error.message).slice(0, 300)}` : '';
-    return new BulkStop(code, `Shopify dataset ${entity} could not be read (${specific})${cause}. Requires Import Operator: fix the cause named here, run \`shopify-extract.js --retry-failed\`, and when the extract completes record the coverage decision \`provided\` for \`shopify:${entity}\`; otherwise show the merchant the row at mapping.`, { details: { query: name, entity, errorCode: target.errorCode || null, coverageId: target.handlerOutcome ? target.handlerOutcome.coverageId : null } });
-  }
-
-  // Runs the outcome's handler for a `settling` entry, writes the terminal state, and returns the
-  // stop the entry causes, or null.
-  async function finishSettling(name) {
-    const target = entry(name);
-    const context = handlerContext(name);
-    let result;
-    if (target.outcome === 'failed') {
-      result = await failedHandlerFor(name)({ ...context, outcome: 'failed' });
-    } else {
-      result = await (handlers.denied || defaultDeniedHandler)({ ...context, outcome: 'denied' });
-    }
-    const terminal = { handlerOutcome: (result && result.handlerOutcome) || {} };
-    if (target.outcome === 'denied') terminal.deniedAt = iso(now());
-    await setState(name, target.outcome, terminal);
-    const handlerStop = result && result.stop ? result.stop : null;
-    if (handlerStop) return stopFor(name, entry(name), handlerStop);
-    if (isStopping(entry(name), caller)) return stopFor(name, entry(name), null);
-    return null;
-  }
-
-  async function settle(name, outcome, details) {
-    await beginSettling(name, outcome, details);
-    const stop = await finishSettling(name);
-    if (stop) throw stop;
-  }
-
-  async function settleAll() {
-    for (const name of manifest.order) {
-      if (entry(name).state === 'settling') {
-        const stop = await finishSettling(name);
-        if (stop) throw stop;
-      }
-    }
-  }
-
-  // -------------------------------------------------------------------------------------------
-  // Session expiry, denial
-  // -------------------------------------------------------------------------------------------
-
-  async function authExpiredStop(extraMessage = '') {
-    const action = await session.recordAuthExpired(projectDir, { stage, store });
-    return new BulkStop('source_auth_expired', `the Shopify connection for ${store} expired during the ${caller}${extraMessage}. Re-approval request ${action.requestId} is pending: run the Shopify session check (with --connect if asked), answer it, then run the same step again. Finished reads are not repeated.`, { details: { requestId: action.requestId } });
-  }
-
-  async function runSessionCheck() {
-    if (typeof sessionCheck !== 'function') return { ok: true };
-    return sessionCheck();
-  }
-
-  async function handleDenial(name, evidence) {
-    const target = entry(name);
-    const check = await runSessionCheck();
-    if (!check.ok && !target.reapproved) {
-      dropOperation(name);
-      await setState(name, 'pending', { reapproved: true });
-      throw await authExpiredStop(` (Shopify denied the ${name} export and the session check failed)`);
-    }
-    const verbatim = { message: 'Shopify refused the bulk export with ACCESS_DENIED', ...evidence };
-    if (DENIAL_ATTRIBUTED_QUERIES.includes(name)) {
-      await settle(name, 'denied', { errorCode: 'ACCESS_DENIED', error: verbatim });
-    } else {
-      await settle(name, 'failed', { errorCode: STOPPING_FAILED_CODE, error: { ...verbatim, message: 'Shopify refused the bulk export with ACCESS_DENIED' } });
-    }
-  }
-
-  // -------------------------------------------------------------------------------------------
-  // Per query
-  // -------------------------------------------------------------------------------------------
-
-  const hasCode = (errors, code) => (errors || []).some((error) => error && error.extensions && error.extensions.code === code);
-
-  async function startQuery(name) {
-    const target = entry(name);
-    await setState(name, 'starting', { startedAt: iso(now()) });
-    progress('Starting Shopify bulk export', { query: name });
-    const result = await execute.startBulkOperation(target.text);
-    if (result.errors && result.errors.length) {
-      if (hasCode(result.errors, 'ACCESS_DENIED')) { await handleDenial(name, { graphqlErrors: result.errors }); return; }
-      await settle(name, 'failed', { errorCode: STOPPING_FAILED_CODE, error: { message: 'Shopify answered the bulk start with GraphQL errors', graphqlErrors: result.errors } });
-      return;
-    }
-    const payload = result.data && result.data.bulkOperationRunQuery;
-    if (!payload) {
-      await settle(name, 'failed', { errorCode: STOPPING_FAILED_CODE, error: { message: 'Shopify answered the bulk start with no bulkOperationRunQuery result' } });
-      return;
-    }
-    if (Array.isArray(payload.userErrors) && payload.userErrors.length) {
-      await settle(name, 'failed', { errorCode: STOPPING_FAILED_CODE, error: { message: 'Shopify refused the bulk start', userErrors: payload.userErrors } });
-      return;
-    }
-    const operation = payload.bulkOperation;
-    if (!operation || typeof operation.id !== 'string') {
-      await settle(name, 'failed', { errorCode: STOPPING_FAILED_CODE, error: { message: 'Shopify answered the bulk start with no operation id' } });
-      return;
-    }
-    const runningAt = now();
-    await setState(name, 'running', { operationId: operation.id, runningAt: iso(runningAt), deadlineAt: iso(runningAt + overdueMs), overdueRound: 1 });
-  }
-
-  function overdueRequestId(name, round) {
-    return session.sequencedRequestId(OVERDUE_NAMESPACE, `${caller}\0${name}\0${entry(name).operationId}`, round);
-  }
-
-  async function readActionIfAny(requestId) {
-    try {
-      return await readUserAction(projectDir, requestId);
-    } catch (error) {
-      if (error.code === 'ENOENT') return null;
-      throw error;
-    }
-  }
-
-  async function raiseOverdue(name) {
-    const target = entry(name);
-    const round = target.overdueRound || 1;
-    const requestId = overdueRequestId(name, round);
-    let action = await readActionIfAny(requestId);
-    if (!action) {
-      action = await createUserAction(projectDir, {
-        requestId,
-        kind: 'question',
-        stage,
-        title: `Shopify export overdue: ${name} (shopify_bulk_overdue)`,
-        prompt: `Shopify has been preparing the \`${name}\` export (operation ${target.operationId}) for over ${round * 4} hours. Wait another 4 hours, or abort this dataset?`,
-        response: { mode: 'choice' },
-        choices: [
-          { id: 'wait', label: 'Wait another 4 hours' },
-          { id: 'abort', label: 'Abort this dataset' },
-        ],
-      });
-    }
-    await setState(name, 'running', { overdueActionId: action.requestId });
-    throw new BulkStop('shopify_bulk_overdue', `Shopify has been preparing the ${name} export for over ${round * 4} hours. Answer request ${action.requestId} (wait or abort), then run the same step again.`, { status: 'awaiting_user_action', details: { query: name, operationId: target.operationId, round, requestIds: [action.requestId] } });
-  }
-
-  // Applies an answered overdue action, or stops while it is unanswered. Returns 'waited' | 'aborted'.
-  async function applyOverdue(name) {
-    const target = entry(name);
-    const action = await readActionIfAny(target.overdueActionId);
-    if (!action || action.status !== 'answered') {
-      throw new BulkStop('shopify_bulk_overdue', `The ${name} export is overdue and request ${target.overdueActionId} is unanswered; nothing was polled or started.`, { status: 'awaiting_user_action', details: { query: name, operationId: target.operationId, round: target.overdueRound || 1, requestIds: [target.overdueActionId] } });
-    }
-    if (action.answer && action.answer.choice === 'abort') {
-      await settle(name, 'failed', { errorCode: 'shopify_bulk_overdue_aborted', error: { message: `The ${name} export was aborted after it ran over ${(target.overdueRound || 1) * 4} hours; Shopify's operation ${target.operationId} was not cancelled and its result was never downloaded`, operationId: target.operationId, requestId: action.requestId } });
-      return 'aborted';
-    }
-    const answeredAt = Date.parse(action.answer && action.answer.answeredAt);
-    const base = Number.isFinite(answeredAt) ? answeredAt : now();
-    await setState(name, 'running', { deadlineAt: iso(base + overdueMs), overdueRound: (target.overdueRound || 1) + 1 }, ['overdueActionId']);
-    return 'waited';
-  }
-
-  function classifyOperation(operation) {
-    if (!operation) return 'failed';
-    if (operation.partialDataUrl) return 'failed';
-    if (operation.status === 'COMPLETED') return 'completed';
-    if (operation.status === 'FAILED' && operation.errorCode === 'ACCESS_DENIED') return 'denied';
-    if (['FAILED', 'CANCELED', 'EXPIRED'].includes(operation.status)) return 'failed';
-    return 'running';
-  }
-
-  async function failedResult(name, { errorCode = STOPPING_FAILED_CODE, message, evidence = {} }) {
-    const target = entry(name);
-    if (target.attempts < MAX_ATTEMPTS) {
-      dropOperation(name);
-      await setState(name, 'pending', { attempts: target.attempts + 1, downloadTries: 0 });
-      return;
-    }
-    await settle(name, 'failed', { errorCode, error: { message, ...evidence } });
-  }
-
-  async function pollOperation(name) {
-    const result = await execute(STATUS_QUERY, { id: entry(name).operationId });
-    if (result.errors && result.errors.length) {
-      return { errors: result.errors };
-    }
-    return { operation: result.data && result.data.node ? result.data.node : null };
-  }
-
-  async function processRunning(name) {
-    if (entry(name).overdueActionId) {
-      const outcome = await applyOverdue(name);
-      if (outcome === 'aborted') return;
-    }
-    let wait = POLL_START_MS;
-    for (;;) {
-      const polled = await pollOperation(name);
-      if (polled.errors) {
-        if (hasCode(polled.errors, 'ACCESS_DENIED')) { await handleDenial(name, { graphqlErrors: polled.errors }); return; }
-        await settle(name, 'failed', { errorCode: STOPPING_FAILED_CODE, error: { message: 'Shopify answered the status poll with GraphQL errors', graphqlErrors: polled.errors } });
-        return;
-      }
-      const operation = polled.operation;
-      const kind = classifyOperation(operation);
-      if (kind === 'running') {
-        if (now() >= Date.parse(entry(name).deadlineAt)) await raiseOverdue(name);
-        await sleep(wait);
-        wait = Math.min(wait * 2, POLL_CAP_MS);
-        continue;
-      }
-      if (kind === 'denied') { await handleDenial(name, { operation: operationEvidence(operation) }); return; }
-      if (kind === 'failed') {
-        await failedResult(name, { message: operation ? `Shopify's bulk operation ended ${operation.status}${operation.errorCode ? ` (${operation.errorCode})` : ''}${operation.partialDataUrl ? ' with only partial data' : ''}` : 'Shopify no longer knows the bulk operation', evidence: { operation: operationEvidence(operation) } });
-        return;
-      }
-      await setState(name, 'running', {
-        url: operation.url || null,
-        fileSize: operation.fileSize === undefined ? null : operation.fileSize,
-        objectCount: operation.objectCount,
-        rootObjectCount: operation.rootObjectCount,
-        createdAt: operation.createdAt,
-        completedAt: operation.completedAt,
-      });
-      await downloadGate(name);
-      return;
-    }
-  }
-
-  function operationEvidence(operation) {
-    if (!operation) return null;
-    return { id: operation.id, status: operation.status, errorCode: operation.errorCode || null, partialDataUrl: operation.partialDataUrl || null, objectCount: operation.objectCount, fileSize: operation.fileSize === undefined ? null : operation.fileSize };
-  }
-
-  // Marks the entry `downloaded` only when the whole gate holds.
-  async function downloadGate(name) {
-    const jsonl = path.join(stagingDir, `${name}.jsonl`);
-    const partial = `${jsonl}.partial`;
-    await fsp.mkdir(stagingDir, { recursive: true });
-    for (;;) {
-      const target = entry(name);
-      if (!target.url) {
-        if (String(target.objectCount) === '0') {
-          await writeTextAtomic(jsonl, '');
-          await setState(name, 'downloaded', { bytes: 0, lines: 0 });
-          return;
-        }
-        await failedResult(name, { message: `Shopify completed the ${name} export with no file for ${target.objectCount} objects`, evidence: { operation: { objectCount: target.objectCount } } });
-        return;
-      }
-      progress('Downloading Shopify bulk file', { query: name });
-      await removeFile(partial);
-      const result = await download(target.url, partial);
-      const written = result && Number.isFinite(result.bytes) ? result.bytes : 0;
-      if (result && (result.status === 400 || result.status === 403)) {
-        await removeFile(partial);
-        if (target.linkReruns >= MAX_LINK_RERUNS) {
-          await settle(name, 'failed', { errorCode: 'shopify_bulk_download_failed', error: { message: `The ${name} export's download link expired ${target.linkReruns + 1} times (HTTP ${result.status})`, httpStatus: result.status } });
-          return;
-        }
-        dropOperation(name);
-        await setState(name, 'pending', { linkReruns: target.linkReruns + 1, downloadTries: 0 });
-        return;
-      }
-      let reason = null;
-      if (!result || result.status !== 200) reason = `the download answered ${result && result.status !== null && result.status !== undefined ? `HTTP ${result.status}` : 'no HTTP response'}${result && result.error ? ` (${result.error})` : ''}`;
-      else if (result.error) reason = `the download failed: ${result.error}`;
-      else if (target.fileSize === null || target.fileSize === undefined || String(written) !== String(target.fileSize)) reason = `the download was ${written} bytes, Shopify reported ${target.fileSize === null || target.fileSize === undefined ? 'no size' : target.fileSize}`;
-      let checked = null;
-      if (!reason) {
-        checked = await validateJsonlFile(partial);
-        if (!checked.ok) reason = checked.reason;
-      }
-      if (!reason) {
-        await fsp.rename(partial, jsonl);
-        await setState(name, 'downloaded', { bytes: written, lines: checked.lines });
-        return;
-      }
-      await removeFile(partial);
-      const tries = target.downloadTries + 1;
-      await setState(name, 'running', { downloadTries: tries });
-      if (tries < MAX_DOWNLOAD_TRIES) continue;
-      await failedResult(name, { errorCode: 'shopify_bulk_download_failed', message: `The ${name} export could not be downloaded whole after ${tries} tries: ${reason}`, evidence: { reason } });
-      return;
-    }
-  }
-
-  async function reassemble(name) {
-    const target = entry(name);
-    const catalog = catalogEntry(name);
-    const jsonl = path.join(stagingDir, `${name}.jsonl`);
-    if (!fs.existsSync(jsonl)) {
-      // `downloaded` promises a complete local file. If it is gone (staging cleared by hand), the
-      // dataset is simply read again: no attempt is spent, nothing is guessed.
-      dropOperation(name);
-      await setState(name, 'pending', { downloadTries: 0 }, ['bytes', 'lines']);
-      return;
-    }
-    const foldedFile = path.join(stagingDir, `${name}.folded.tmp`);
-    const folded = await foldBulkJsonl(jsonl, catalog, { outputFile: foldedFile });
-    if (folded.orphans) {
-      await settle(name, 'failed', { errorCode: 'shopify_bulk_orphan_child', error: { message: `${folded.orphanCount} child row(s) of ${name} match no parent row`, orphans: folded.orphans } });
-      return;
-    }
-    const hook = handlers.reassemble && handlers.reassemble[name];
-    let records = null;
-    let recordsFile = null;
-    let recordCount = folded.recordCount;
-    let outputs = {};
-    let sidecarFiles = {};
-    let result = {};
-    let outputFile = null;
-    let entryFields = {};
-    let hookStop = null;
-    if (hook) {
-      const hooked = await hook({ ...handlerContext(name), records: iterateNdjson(foldedFile), recordsFile: foldedFile, recordCount, lines: folded.lines });
-      if (hooked && hooked.ok === false) {
-        await removeFile(foldedFile);
-        await settle(name, 'failed', { errorCode: hooked.errorCode || STOPPING_FAILED_CODE, error: { message: hooked.message || `The ${name} recheck failed (${hooked.errorCode || STOPPING_FAILED_CODE})`, ...(hooked.evidence ? { evidence: hooked.evidence } : {}) } });
-        return;
-      }
-      if (hooked && hooked.records) records = hooked.records;
-      if (hooked && hooked.recordsFile) recordsFile = hooked.recordsFile;
-      if (hooked && Number.isInteger(hooked.recordCount)) recordCount = hooked.recordCount;
-      if (hooked && hooked.sidecars) outputs = hooked.sidecars;
-      if (hooked && hooked.sidecarFiles) sidecarFiles = hooked.sidecarFiles;
-      if (hooked && hooked.manifest) Object.assign(manifest, hooked.manifest);
-      if (hooked && hooked.result) result = hooked.result;
-      if (hooked && hooked.outputFile) outputFile = hooked.outputFile;
-      if (hooked && hooked.entry) entryFields = hooked.entry;
-      if (hooked && hooked.stop) hookStop = hooked.stop;
-    }
-    const readAt = { createdAt: target.createdAt || null, completedAt: target.completedAt || null };
-    const durationMs = Number.isFinite(Date.parse(target.completedAt)) && Number.isFinite(Date.parse(target.createdAt)) ? Date.parse(target.completedAt) - Date.parse(target.createdAt) : null;
-    if (extract) {
-      const stagedName = outputFile || `${name}.ndjson`;
-      const staged = path.join(stagingDir, stagedName);
-      if (recordsFile) {
-        await removeFile(staged);
-        await fsp.rename(recordsFile, staged);
-        await removeFile(foldedFile);
-      } else if (records) {
-        recordCount = await writeNdjsonAtomic(staged, records);
-        await removeFile(foldedFile);
-      } else {
-        await fsp.rename(foldedFile, staged);
-      }
-      const sidecarNames = [...new Set([...Object.keys(outputs), ...Object.keys(sidecarFiles)])].sort();
-      const oldSidecars = Array.isArray(target.sidecars) ? target.sidecars : [];
-      for (const sidecar of oldSidecars.filter((sidecar) => !sidecarNames.includes(sidecar))) {
-        await removeFile(path.join(stagingDir, `sidecar-${sidecar}.ndjson`));
-        manifest.obsoleteSidecars = [...new Set([...(manifest.obsoleteSidecars || []), sidecar])].sort();
-      }
-      for (const sidecar of sidecarNames) {
-        const destination = path.join(stagingDir, `sidecar-${sidecar}.ndjson`);
-        if (sidecarFiles[sidecar]) {
-          await removeFile(destination);
-          await fsp.rename(sidecarFiles[sidecar], destination);
-        } else {
-          await writeNdjsonAtomic(destination, outputs[sidecar]);
-        }
-      }
-      manifest.sidecars = [...new Set(manifest.order.flatMap((query) => query === name ? sidecarNames : (entry(query).sidecars || [])))].sort();
-      await setState(name, 'done', { readAt, durationMs, records: recordCount, sidecars: sidecarNames, ...(outputFile ? { outputFile } : {}), ...entryFields }, ['url']);
-    } else {
-      await removeFile(foldedFile);
-      await setState(name, 'done', { readAt, durationMs, result: { records: recordCount, ...result }, ...entryFields }, ['url']);
-    }
-    await removeFile(jsonl);
-    if (hookStop) throw new BulkStop(hookStop.code, hookStop.message || `Shopify extract stopped (${hookStop.code}) at ${name}.`, { details: { query: name, entity: entityOf(name), ...(hookStop.details || {}) } });
-  }
-
-  async function processQuery(name) {
-    for (;;) {
-      const target = entry(name);
-      switch (target.state) {
-        case 'done':
-          return;
-        case 'denied':
-          if (isStopping(target, caller)) throw stopFor(name, target, null, { later: true });
-          return;
-        case 'failed':
-          if (extract) throw stopFor(name, target, null, { later: true });
-          return;
-        case 'settling': {
-          const stop = await finishSettling(name);
-          if (stop) throw stop;
-          break;
-        }
-        case 'starting':
-          dropOperation(name);
-          await setState(name, 'pending');
-          break;
-        case 'pending':
-          await startQuery(name);
-          break;
-        case 'running':
-          await processRunning(name);
-          break;
-        case 'downloaded':
-          await reassemble(name);
-          break;
-        default:
-          throw new Error(`unknown query state ${target.state} for ${name}`);
-      }
-    }
-  }
-
-  // -------------------------------------------------------------------------------------------
-  // Resume resets
-  // -------------------------------------------------------------------------------------------
-
-  async function cleanLeftovers() {
-    let names = [];
-    try {
-      names = await fsp.readdir(stagingDir);
-    } catch (error) {
-      if (error.code !== 'ENOENT') throw error;
-    }
-    for (const leftover of names.filter((file) => file.endsWith('.partial') || file.endsWith('.tmp'))) await removeFile(path.join(stagingDir, leftover));
-    // A crash between the `done` write and the JSONL delete leaves the file behind.
-    for (const name of manifest.order) if (entry(name).state === 'done') await removeFile(path.join(stagingDir, `${name}.jsonl`));
-  }
-
-  function resetEntry(name, fields = {}) {
-    const target = entry(name);
-    dropOperation(name);
-    for (const field of ['outcome', 'errorCode', 'error', 'settlingAt', 'handlerOutcome', 'deniedAt', 'readAt', 'durationMs', 'records', 'result', 'bytes', 'lines']) delete target[field];
-    Object.assign(target, { state: 'pending', attempts: 0, downloadTries: 0, linkReruns: 0, reapproved: false }, fields);
-  }
-
-  async function applyResets() {
-    if (!extract) return;
-    let changed = false;
-    if (typeof deniedReset === 'function') {
-      for (const reset of (await deniedReset({ manifest, projectDir })) || []) {
-        resetEntry(reset.name, { text: reset.text });
-        changed = true;
-      }
-    }
-    if (retryFailed) {
-      for (const name of manifest.order) {
-        if (isStopping(entry(name), caller)) { resetEntry(name); changed = true; }
-      }
-      if (manifest.countFailure && manifest.countFailure.entities) {
-        for (const failedEntity of manifest.countFailure.entities) {
-          for (const name of manifest.order.filter((candidate) => entityOf(candidate) === failedEntity)) {
-            await removeFile(path.join(stagingDir, `${name}.ndjson`));
-            resetEntry(name);
-          }
-        }
-        delete manifest.countFailure;
-        changed = true;
-      }
-    }
-    // A `done` extract entry promises its staged output, and a resume never reads a
-    // finished query again. When that output is gone before the final join, read the query again
-    // without spending an attempt instead of joining the hole as an empty dataset.
-    for (const name of manifest.order) {
-      const target = entry(name);
-      if (target.state !== 'done') continue;
-      const files = [target.outputFile || `${name}.ndjson`, ...(target.sidecars || []).map((sidecar) => `sidecar-${sidecar}.ndjson`)].map((file) => path.join(stagingDir, file));
-      const present = await Promise.all(files.map((file) => fsp.access(file).then(() => true, (error) => { if (error.code === 'ENOENT') return false; throw error; })));
-      if (present.every(Boolean)) continue;
-      for (const file of files) await removeFile(file);
-      resetEntry(name);
-      changed = true;
-    }
-    if (changed) await save('resets');
-  }
-
-  // -------------------------------------------------------------------------------------------
-  // Extract: the final join and the count check
-  // -------------------------------------------------------------------------------------------
-
-  function expectedFinalFiles() {
-    const entities = [...new Set(manifest.order.map(entityOf))];
-    return [
-      ...entities.map((entity) => {
-        const first = manifest.order.find((name) => entityOf(name) === entity);
-        return (first && entry(first).outputFile) || `${entity}.ndjson`;
-      }),
-      ...(manifest.sidecars || []).map((entity) => `${entity}.ndjson`),
-    ];
-  }
-
-  async function joinFinalFiles() {
-    await fsp.mkdir(outputDir, { recursive: true });
-    const staged = (name) => path.join(stagingDir, `${name}.ndjson`);
-    let droppedOrphans = 0;
-    const droppedByQuery = {};
-    for (const entity of [...new Set(manifest.order.map(entityOf))]) {
-      const members = manifest.order.filter((name) => entityOf(name) === entity);
-      if (entity !== 'orders') {
-        const outputFile = entry(members[0]).outputFile || `${entity}.ndjson`;
-        const stagedFile = entry(members[0]).outputFile || `${members[0]}.ndjson`;
-        await copyFileAtomic(path.join(stagingDir, stagedFile), path.join(outputDir, outputFile));
-        continue;
-      }
-      const joined = await joinOrderFiles({
-        coreFile: staged('orders-core'),
-        refundsFile: members.includes('orders-refunds') ? staged('orders-refunds') : null,
-        fulfillmentsFile: members.includes('orders-fulfillments') ? staged('orders-fulfillments') : null,
-        outputFile: path.join(outputDir, 'orders.ndjson'),
-        workDir: stagingDir,
-      });
-      droppedOrphans += joined.droppedOrphans;
-      Object.assign(droppedByQuery, joined.droppedByQuery);
-    }
-    for (const sidecar of manifest.sidecars || []) await copyFileAtomic(path.join(stagingDir, `sidecar-${sidecar}.ndjson`), path.join(outputDir, `${sidecar}.ndjson`));
-    const currentSidecars = new Set(manifest.sidecars || []);
-    for (const sidecar of new Set([...(manifest.previousSidecars || []), ...(manifest.obsoleteSidecars || [])])) {
-      if (!currentSidecars.has(sidecar)) await removeFile(path.join(outputDir, `${sidecar}.ndjson`));
-    }
-    manifest.droppedOrphans = droppedOrphans;
-    manifest.droppedOrphansByQuery = droppedByQuery;
-  }
-
-  const COUNT_QUERIES = {
-    customers: (snapshot) => `query RpExtractCustomersCount { customersCount(query: "created_at:<='${snapshot}'", limit: null) { count precision } }`,
-    orders: (snapshot) => `query RpExtractOrdersCount { ordersCount(query: "created_at:<='${snapshot}'", limit: null) { count precision } }`,
-  };
-
-  async function readExactCount(entity) {
-    const root = entity === 'customers' ? 'customersCount' : 'ordersCount';
-    let result;
-    try {
-      result = await execute(COUNT_QUERIES[entity](manifest.snapshotTime));
-    } catch (error) {
-      if (error instanceof ShopifyCliError && error.code === 'shopify_throttled') return { error: { code: 'shopify_throttled', message: error.message } };
-      throw error;
-    }
-    if (result.errors && result.errors.length) return { error: { code: 'GRAPHQL_ERROR', message: result.errors.map((error) => error && error.message).filter(Boolean).join('; ').slice(0, 300) } };
-    const counted = result.data && result.data[root];
-    if (!counted || !Number.isInteger(counted.count)) return { error: { code: 'COUNT_MISSING', message: 'the count read returned no count' } };
-    return { count: counted.count, precision: counted.precision || null };
-  }
-
-  async function countCheck() {
-    const customersReceived = (await countLines(path.join(outputDir, 'customers.ndjson'))) || 0;
-    const withheld = (manifest.sidecars || []).includes('customers-withheld-identity') ? ((await countLines(path.join(outputDir, 'customers-withheld-identity.ndjson'))) || 0) : 0;
-    const received = { customers: customersReceived + withheld, orders: (await countLines(path.join(outputDir, 'orders.ndjson'))) || 0 };
-    manifest.counts = {};
-    const rows = [];
-    const stopCodes = [];
-    for (const entity of ['customers', 'orders']) {
-      const expected = await readExactCount(entity);
-      const record = { received: received[entity], ...(entity === 'customers' ? { withheldIdentity: withheld } : {}) };
-      let outcome;
-      if (expected.error) {
-        outcome = 'inexact';
-        Object.assign(record, { error: expected.error });
-      } else if (expected.precision !== 'EXACT') {
-        outcome = 'inexact';
-        Object.assign(record, { expected: expected.count, precision: expected.precision });
-      } else if (received[entity] < expected.count) {
-        outcome = 'short';
-        Object.assign(record, { expected: expected.count, precision: expected.precision });
-      } else {
-        outcome = 'pass';
-        Object.assign(record, { expected: expected.count, precision: expected.precision });
-      }
-      record.outcome = outcome;
-      manifest.counts[entity] = record;
-      if (outcome === 'pass') continue;
-      const code = outcome === 'short' ? 'shopify_extract_count_short' : 'shopify_extract_count_inexact';
-      const pointer = outcome === 'short'
-        ? `counts.${entity}: received ${record.received} < Shopify's exact count ${record.expected} (precision ${record.precision})`
-        : `counts.${entity}: Shopify's count is not exact (${record.error ? `${record.error.code}: ${record.error.message}` : `precision ${record.precision}`}); received ${record.received}`;
-      rows.push(coverageRowFor(orderQueryOf(entity), { artifact: `${RAW_EXTRACT_DIR}/${EXTRACT_MANIFEST_FILE}`, pointer, recoveryAction: `Run shopify-extract.js --retry-failed to read ${entity} again, or decide the shortfall at mapping.` }));
-      stopCodes.push({ code, entity });
-    }
-    if (!stopCodes.length) { await save('counts'); return; }
-    manifest.countFailure = { entities: stopCodes.map((stop) => stop.entity), codes: stopCodes.map((stop) => stop.code) };
-    await save('counts');
-    const result = await upsertAdapterCoverageRows(projectDir, { adapter: 'shopify', rows });
-    throw new ShopifyExtractStop(stopCodes[0].code, `Shopify extract stopped (${stopCodes[0].code}): ${rows.map((row) => row.coverageId).join(', ')} recorded as source-coverage row(s) for the merchant's decision at mapping. Counts: ${JSON.stringify(manifest.counts)}. Requires Import Operator: after any cause is fixed, run \`shopify-extract.js --retry-failed\` to read the dataset again; when the extract completes, record the coverage decision \`provided\` for each row, otherwise show the merchant the row at mapping.`, { coverageIds: rows.map((row) => row.coverageId), ...result });
-  }
-
-  function orderQueryOf(entity) {
-    return entity === 'customers' ? 'customers' : 'orders-core';
-  }
-
-  async function finishExtract() {
-    if (!manifest.order.every((name) => entry(name).state === 'done')) {
-      const open = manifest.order.filter((name) => entry(name).state !== 'done');
-      throw new BulkStop(STOPPING_FAILED_CODE, `The extract cannot finish: ${open.join(', ')} ${open.length === 1 ? 'is' : 'are'} not read (${open.map((name) => `${name} ${entry(name).state}`).join(', ')}). Nothing was marked complete.`, { details: { open } });
-    }
-    await joinFinalFiles();
-    await save('joined');
-    await countCheck();
-    manifest.status = 'complete';
-    manifest.completedAt = iso(now());
-    delete manifest.countFailure;
-    await save('complete');
-    await fsp.rm(stagingDir, { recursive: true, force: true });
-  }
-
-  // -------------------------------------------------------------------------------------------
-  // run()
-  // -------------------------------------------------------------------------------------------
-
-  async function filesPresent() {
-    for (const file of expectedFinalFiles()) {
-      try {
-        await fsp.access(path.join(outputDir, file));
-      } catch {
-        return false;
-      }
-    }
-    return true;
-  }
-
-  async function runInner() {
-    manifest = await readManifest();
-    let storeMatches = !manifest;
-    if (manifest && manifest.store) {
-      try {
-        storeMatches = session.normalizeDomain(manifest.store) === normalizedStore;
-      } catch {
-        storeMatches = false;
-      }
-    }
-    if (manifest && !storeMatches) {
-      const recorded = manifest.store || null;
-      manifest = null;
-      throw new BulkStop('shopify_extract_store_mismatch', `${path.basename(manifestPath)} was read from ${recorded || 'an unrecorded store'}, not ${normalizedStore}. Nothing was read or changed.`, { details: { recorded, store: normalizedStore } });
-    }
-    if (!manifest) {
-      if (!Array.isArray(queries) || !queries.length) throw new Error('queries are required to start a new manifest');
-      manifest = newManifest({ caller, store: normalizedStore, snapshotTime: extract ? snapshotTime : null, queries, previousSidecars, now });
-      await save('created');
-    }
-    if (extract && manifest.status === 'complete') {
-      if (!(await filesPresent())) {
-        throw new BulkStop('shopify_extract_files_missing', `The extract manifest says complete but its final files are missing from ${path.relative(projectDir, outputDir)}. Run shopify-extract.js --catch-up to read the store again.`);
-      }
-      await fsp.rm(stagingDir, { recursive: true, force: true });
-      return { status: 'complete', skipped: true };
-    }
-    if (manifest.lastStop) { delete manifest.lastStop; await save('cleared-stop'); }
-    await settleAll();
-    await cleanLeftovers();
-    await applyResets();
-    for (const name of manifest.order) {
-      if (isStopping(entry(name), caller)) throw stopFor(name, entry(name), null, { later: true });
-    }
-    for (const name of manifest.order) await processQuery(name);
-    if (extract) await finishExtract();
-    else { manifest.status = 'complete'; manifest.completedAt = iso(now()); await save('complete'); }
-    return { status: 'complete' };
-  }
-
-  async function run() {
-    let stop = null;
-    try {
-      const outcome = await runInner();
-      return { ...outcome, manifest };
-    } catch (error) {
-      if (error instanceof BulkStop) stop = error;
-      else if (error instanceof ShopifyExtractStop) stop = new BulkStop(error.code, error.message, { details: error.details });
-      else if (error instanceof ShopifyCliError && error.code === 'source_auth_expired') stop = await authExpiredStop();
-      else if (error instanceof ShopifyCliError) stop = new BulkStop(error.code, error.message, { details: { cli: true } });
-      else throw error;
-    }
-    if (manifest) {
-      manifest.lastStop = { code: stop.code, message: stop.message, status: stop.status, at: iso(now()) };
-      await save('stopped');
-    }
-    return { status: stop.status, code: stop.code, message: stop.message, details: stop.details, manifest };
-  }
-
-  return { run, get manifest() { return manifest; } };
-}
-
-module.exports = {
-  QUERIES_DIR,
-  QUERY_CATALOG,
-  QUERY_NAMES,
-  CUSTOMER_QUERIES,
-  ORDER_QUERIES,
-  DENIAL_ATTRIBUTED_QUERIES,
-  MANIFEST_SCHEMA_VERSION,
-  EXTRACT_MANIFEST_FILE,
-  RAW_EXTRACT_DIR,
-  SNAPSHOT_PLACEHOLDER,
-  STATUS_QUERY,
-  POLL_START_MS,
-  POLL_CAP_MS,
-  OVERDUE_MS,
-  MAX_DOWNLOAD_TRIES,
-  MAX_LINK_RERUNS,
-  BulkStop,
-  catalogEntry,
-  entityOf,
-  readQueryFile,
-  generateQueryText,
-  connectionShape,
-  validateJsonlFile,
-  foldBulkJsonl,
-  joinOrderFiles,
-  defaultDownload,
-  registerHandlers,
-  clearRegisteredHandlers,
-  registeredProjection: () => registered.project,
-  createBulkRunner,
-};