@wix/agent-skills 1.15.0 → 1.16.0

package/skills/wix-replatform/resources/rp-target-wix/lib/domain-knowledge.js

@@ -0,0 +1,384 @@
+'use strict';
+
+const fs = require('node:fs');
+const path = require('node:path');
+const wixWriters = require('./wix-writers.js');
+
+const CLASSIFICATIONS = new Set([
+  'native',
+  'cms',
+  'native-plus-cms',
+  'setup-config',
+  'skip-by-default',
+  'unsupported-native-gap',
+]);
+const ID_POLICIES = new Set(['client-assigned', 'server-assigned', 'natural-key', 'not-applicable']);
+const VERIFICATIONS = new Set(['verified-live', 'docs', 'source-review', 'internal-only', 'unverified', 'none']);
+const RELIABILITY = new Set(['reliable', 'partially-reliable', 'unreliable', 'unknown']);
+const SAFE_MODE_CONTACT_KINDS = new Set(['email', 'phone']);
+const WRITER_IDS = new Set(Object.keys(wixWriters).filter((name) => typeof wixWriters[name] === 'function'));
+
+function knowledgeRoot(rootDir = path.resolve(__dirname, '..')) {
+  return path.join(rootDir, 'domains');
+}
+
+function readJson(filePath) {
+  return JSON.parse(fs.readFileSync(filePath, 'utf8'));
+}
+
+function writeJson(filePath, value) {
+  fs.writeFileSync(filePath, `${JSON.stringify(value, null, 2)}\n`);
+}
+
+function entityRef(entity) {
+  return `${entity.domain}/${entity.entity}`;
+}
+
+function listDomainDirs(domainsDir) {
+  return fs
+    .readdirSync(domainsDir, { withFileTypes: true })
+    .filter((entry) => entry.isDirectory())
+    .map((entry) => entry.name)
+    .sort();
+}
+
+function listEntityFiles(domainDir) {
+  const entitiesDir = path.join(domainDir, 'entities');
+  if (!fs.existsSync(entitiesDir)) return [];
+  return fs
+    .readdirSync(entitiesDir, { withFileTypes: true })
+    .filter((entry) => entry.isFile() && entry.name.endsWith('.json'))
+    .map((entry) => entry.name)
+    .sort();
+}
+
+function loadDomain(domainsDir, domain) {
+  return readJson(path.join(domainsDir, domain, 'domain.json'));
+}
+
+function loadEntity(domainsDir, domain, entity) {
+  return readJson(path.join(domainsDir, domain, 'entities', `${entity}.json`));
+}
+
+function generateIndex(domainsDir) {
+  const index = {
+    schemaVersion: 1,
+    domains: {},
+    sourceAliasIndex: {},
+    routeAliasIndex: {},
+    flags: {},
+  };
+
+  for (const domain of listDomainDirs(domainsDir)) {
+    const domainPath = path.join(domain, 'domain.json');
+    const domainJson = loadDomain(domainsDir, domain);
+    const entities = {};
+    for (const fileName of listEntityFiles(path.join(domainsDir, domain))) {
+      const entityId = fileName.replace(/\.json$/, '');
+      const entityPath = path.join(domain, 'entities', fileName);
+      const entity = readJson(path.join(domainsDir, entityPath));
+      entities[entityId] = {
+        path: `domains/${entityPath}`,
+        displayName: entity.displayName,
+        classification: entity.target && entity.target.classification,
+        reliabilityStatus: entity.reliability && entity.reliability.status,
+        reliabilityFlags: (entity.reliability && entity.reliability.flags) || [],
+        summary: firstPitfallOrGuidance(entity),
+      };
+      for (const alias of entity.sourceAliases || []) {
+        const key = `${alias.sourceSystem}:${alias.sourceEntity}`;
+        if (!index.sourceAliasIndex[key]) index.sourceAliasIndex[key] = [];
+        index.sourceAliasIndex[key].push(`${domain}/${entityId}`);
+        for (const route of alias.routes || []) {
+          if (!index.routeAliasIndex[route]) index.routeAliasIndex[route] = [];
+          index.routeAliasIndex[route].push(`${domain}/${entityId}`);
+        }
+      }
+      for (const flag of (entity.reliability && entity.reliability.flags) || []) {
+        if (!index.flags[flag]) index.flags[flag] = [];
+        index.flags[flag].push(`${domain}/${entityId}`);
+      }
+    }
+    index.domains[domain] = {
+      path: `domains/${domainPath}`,
+      displayName: domainJson.displayName,
+      ownerHint: domainJson.ownerHint,
+      entities,
+    };
+  }
+
+  for (const collection of [index.sourceAliasIndex, index.routeAliasIndex, index.flags]) {
+    for (const key of Object.keys(collection)) collection[key] = Array.from(new Set(collection[key])).sort();
+  }
+
+  return index;
+}
+
+function firstPitfallOrGuidance(entity) {
+  if (Array.isArray(entity.pitfalls) && entity.pitfalls[0] && entity.pitfalls[0].summary) {
+    return entity.pitfalls[0].summary;
+  }
+  if (Array.isArray(entity.mappingGuidance) && entity.mappingGuidance[0]) return entity.mappingGuidance[0];
+  return '';
+}
+
+function validateKnowledge(domainsDir) {
+  const errors = [];
+  const indexPath = path.join(domainsDir, 'index.json');
+  const index = fs.existsSync(indexPath) ? readJson(indexPath) : null;
+  const generated = generateIndex(domainsDir);
+
+  if (!fs.existsSync(path.join(domainsDir, 'schema.json'))) {
+    errors.push('domains/schema.json is missing');
+  }
+
+  for (const domain of listDomainDirs(domainsDir)) {
+    const domainFile = path.join(domainsDir, domain, 'domain.json');
+    if (!fs.existsSync(domainFile)) {
+      errors.push(`${domain}: missing domain.json`);
+      continue;
+    }
+    const domainJson = readJson(domainFile);
+    requireFields(domainJson, ['schemaVersion', 'domain', 'displayName', 'ownerHint', 'defaultImportOrder', 'evidence'], `${domain}/domain.json`, errors);
+    if (domainJson.schemaVersion !== 1) errors.push(`${domain}/domain.json: schemaVersion must be 1`);
+    if (domainJson.domain !== domain) errors.push(`${domain}/domain.json: domain must match directory`);
+    for (const entityId of domainJson.defaultImportOrder || []) {
+      if (!fs.existsSync(path.join(domainsDir, domain, 'entities', `${entityId}.json`))) {
+        errors.push(`${domain}/domain.json: defaultImportOrder references missing entity ${entityId}`);
+      }
+    }
+    validateEvidence(domainJson.evidence || [], `${domain}/domain.json`, errors);
+
+    for (const fileName of listEntityFiles(path.join(domainsDir, domain))) {
+      const entityId = fileName.replace(/\.json$/, '');
+      const entity = loadEntity(domainsDir, domain, entityId);
+      const label = `${domain}/entities/${fileName}`;
+      validateEntity(entity, domain, entityId, label, errors);
+    }
+  }
+
+  if (!index) {
+    errors.push('domains/index.json is missing; run domain-knowledge-validate.js --write-index');
+  } else {
+    const current = JSON.stringify(index);
+    const expected = JSON.stringify(generated);
+    if (current !== expected) {
+      errors.push('domains/index.json is stale; run domain-knowledge-validate.js --write-index');
+    }
+    validateIndexConsistency(index, domainsDir, errors);
+  }
+
+  return { ok: errors.length === 0, errors, generatedIndex: generated };
+}
+
+function requireFields(value, fields, label, errors) {
+  for (const field of fields) {
+    if (value[field] === undefined) errors.push(`${label}: missing required field ${field}`);
+  }
+}
+
+function validateEntity(entity, domain, entityId, label, errors) {
+  requireFields(entity, ['schemaVersion', 'domain', 'entity', 'displayName', 'target', 'sourceAliases', 'preferredWrite', 'reliability', 'pitfalls', 'mappingGuidance', 'evidence'], label, errors);
+  if (entity.schemaVersion !== 1) errors.push(`${label}: schemaVersion must be 1`);
+  if (entity.domain !== domain) errors.push(`${label}: domain must match file path`);
+  if (entity.entity !== entityId) errors.push(`${label}: entity must match file path`);
+  if (!CLASSIFICATIONS.has(entity.target && entity.target.classification)) errors.push(`${label}: invalid target.classification`);
+  if (!ID_POLICIES.has(entity.target && entity.target.idPolicy)) errors.push(`${label}: invalid target.idPolicy`);
+  if (typeof (entity.target && entity.target.crosswalkRequired) !== 'boolean') errors.push(`${label}: target.crosswalkRequired must be boolean`);
+  if (!VERIFICATIONS.has(entity.preferredWrite && entity.preferredWrite.verification)) errors.push(`${label}: invalid preferredWrite.verification`);
+  if (!RELIABILITY.has(entity.reliability && entity.reliability.status)) errors.push(`${label}: invalid reliability.status`);
+  if (!Array.isArray(entity.reliability && entity.reliability.flags)) errors.push(`${label}: reliability.flags must be an array`);
+  if (entity.preferredWrite && entity.preferredWrite.writerId !== null && !WRITER_IDS.has(entity.preferredWrite.writerId)) {
+    errors.push(`${label}: writerId ${entity.preferredWrite.writerId} is not exported by wix-writers.js`);
+  }
+  if (entity.target && entity.target.classification !== 'cms' && entity.target.idPolicy === 'server-assigned' && entity.target.crosswalkRequired !== true) {
+    errors.push(`${label}: server-assigned non-CMS targets must require crosswalk`);
+  }
+  for (const alias of entity.sourceAliases || []) {
+    if (!alias.sourceSystem || !alias.sourceEntity) errors.push(`${label}: sourceAliases entries must include sourceSystem and sourceEntity`);
+  }
+  validateSafeModeContactFields(entity.safeModeContactFields, label, errors);
+  validateEvidence(entity.evidence || [], label, errors);
+}
+
+function validateSafeModeContactFields(fields, label, errors) {
+  if (fields === undefined) return;
+  if (!Array.isArray(fields)) {
+    errors.push(`${label}: safeModeContactFields must be an array when present`);
+    return;
+  }
+  for (const [index, field] of fields.entries()) {
+    const fieldLabel = `${label}: safeModeContactFields[${index}]`;
+    if (!field || typeof field !== 'object' || Array.isArray(field)) {
+      errors.push(`${fieldLabel} must be an object`);
+      continue;
+    }
+    if (!SAFE_MODE_CONTACT_KINDS.has(field.kind)) errors.push(`${fieldLabel}.kind must be email or phone`);
+    if (!field.targetPath || typeof field.targetPath !== 'string') errors.push(`${fieldLabel}.targetPath must be a non-empty string`);
+    if (!field.source || typeof field.source !== 'string') errors.push(`${fieldLabel}.source must be a non-empty string`);
+  }
+}
+
+function validateEvidence(items, label, errors) {
+  if (!Array.isArray(items) || items.length === 0) {
+    errors.push(`${label}: evidence must be a non-empty array`);
+    return;
+  }
+  for (const item of items) {
+    if (!item.url && !item.path) errors.push(`${label}: every evidence item must include url or path`);
+  }
+}
+
+function validateIndexConsistency(index, domainsDir, errors) {
+  const flagged = new Set(index.flags && index.flags.IMPORT_UNRELIABLE ? index.flags.IMPORT_UNRELIABLE : []);
+  const entityFlagged = new Set();
+
+  for (const [domain, domainEntry] of Object.entries(index.domains || {})) {
+    if (!fs.existsSync(path.join(domainsDir, domain, 'domain.json'))) errors.push(`index: missing domain file for ${domain}`);
+    for (const [entityId, entityEntry] of Object.entries(domainEntry.entities || {})) {
+      const ref = `${domain}/${entityId}`;
+      const entityPath = path.join(domainsDir, domain, 'entities', `${entityId}.json`);
+      if (!fs.existsSync(entityPath)) {
+        errors.push(`index: missing entity file for ${ref}`);
+        continue;
+      }
+      const entity = readJson(entityPath);
+      if ((entity.reliability.flags || []).includes('IMPORT_UNRELIABLE')) entityFlagged.add(ref);
+      if (entityEntry.path !== `domains/${domain}/entities/${entityId}.json`) {
+        errors.push(`index: invalid path for ${ref}`);
+      }
+    }
+  }
+
+  for (const ref of entityFlagged) {
+    if (!flagged.has(ref)) errors.push(`index: ${ref} has IMPORT_UNRELIABLE but is missing from flags`);
+  }
+  for (const ref of flagged) {
+    if (!entityFlagged.has(ref)) errors.push(`index: ${ref} is flagged IMPORT_UNRELIABLE but entity file is not`);
+  }
+}
+
+function loadIndex(domainsDir) {
+  return readJson(path.join(domainsDir, 'index.json'));
+}
+
+function listDomains(domainsDir) {
+  const index = loadIndex(domainsDir);
+  return Object.entries(index.domains || {}).map(([domain, info]) => ({
+    domain,
+    displayName: info.displayName,
+    ownerHint: info.ownerHint,
+    path: info.path,
+  }));
+}
+
+function listEntities(domainsDir, domain) {
+  const index = loadIndex(domainsDir);
+  const domainInfo = index.domains && index.domains[domain];
+  if (!domainInfo) throw new Error(`Unknown domain: ${domain}`);
+  return Object.entries(domainInfo.entities || {}).map(([entity, info]) => ({
+    ref: `${domain}/${entity}`,
+    entity,
+    displayName: info.displayName,
+    classification: info.classification,
+    reliabilityStatus: info.reliabilityStatus,
+    reliabilityFlags: info.reliabilityFlags,
+    path: info.path,
+  }));
+}
+
+function readEntityByRef(domainsDir, ref) {
+  const [domain, entity] = ref.split('/');
+  if (!domain || !entity) throw new Error(`Invalid ref: ${ref}`);
+  return loadEntity(domainsDir, domain, entity);
+}
+
+function resolveSource(domainsDir, { sourceSystem, sourceEntity, route }) {
+  const index = loadIndex(domainsDir);
+  const refs = new Set();
+  if (sourceSystem && sourceEntity) {
+    for (const ref of index.sourceAliasIndex[`${sourceSystem}:${sourceEntity}`] || []) refs.add(ref);
+  }
+  if (route) {
+    for (const [pattern, patternRefs] of Object.entries(index.routeAliasIndex || {})) {
+      if (route === pattern || routeMatches(pattern, route)) {
+        for (const ref of patternRefs) refs.add(ref);
+      }
+    }
+  }
+  return Array.from(refs).sort().map((ref) => {
+    const entity = readEntityByRef(domainsDir, ref);
+    return {
+      ref,
+      confidence: bestAliasConfidence(entity, { sourceSystem, sourceEntity, route }),
+      sourceAliases: (entity.sourceAliases || []).filter((alias) => aliasMatches(alias, { sourceSystem, sourceEntity, route })),
+    };
+  });
+}
+
+function routeMatches(pattern, route) {
+  if (!pattern.includes('{') && !pattern.includes(':')) return false;
+  const escaped = pattern.replace(/[.*+?^${}()|[\]\\]/g, '\\$&').replace(/\\\{[^}]+\\\}/g, '[^/]+').replace(/:[A-Za-z0-9_-]+/g, '[^/]+');
+  return new RegExp(`^${escaped}$`).test(route);
+}
+
+function aliasMatches(alias, query) {
+  const sourceMatches = (!query.sourceSystem || alias.sourceSystem === query.sourceSystem) && (!query.sourceEntity || alias.sourceEntity === query.sourceEntity);
+  const routeMatchesAlias = !query.route || (alias.routes || []).some((pattern) => query.route === pattern || routeMatches(pattern, query.route));
+  return sourceMatches && routeMatchesAlias;
+}
+
+function bestAliasConfidence(entity, query) {
+  const order = { high: 3, medium: 2, low: 1 };
+  let best = 'low';
+  for (const alias of entity.sourceAliases || []) {
+    if (aliasMatches(alias, query) && (order[alias.confidence] || 0) > (order[best] || 0)) best = alias.confidence;
+  }
+  return best;
+}
+
+function listFlagged(domainsDir, flag) {
+  const index = loadIndex(domainsDir);
+  return (index.flags && index.flags[flag] ? index.flags[flag] : []).map((ref) => {
+    const entity = readEntityByRef(domainsDir, ref);
+    return {
+      ref,
+      displayName: entity.displayName,
+      classification: entity.target.classification,
+      reliabilityStatus: entity.reliability.status,
+      summary: firstPitfallOrGuidance(entity),
+    };
+  });
+}
+
+function summarizeEntities(domainsDir, refs, { includeEvidence = false } = {}) {
+  return refs.map((ref) => {
+    const entity = readEntityByRef(domainsDir, ref);
+    const summary = {
+      ref,
+      displayName: entity.displayName,
+      target: entity.target,
+      preferredWrite: entity.preferredWrite,
+      reliability: entity.reliability,
+      pitfalls: entity.pitfalls,
+      mappingGuidance: entity.mappingGuidance,
+      setupRequirements: entity.setupRequirements || [],
+      fieldContracts: entity.fieldContracts || [],
+    };
+    if (includeEvidence) summary.evidence = entity.evidence;
+    return summary;
+  });
+}
+
+module.exports = {
+  knowledgeRoot,
+  generateIndex,
+  validateKnowledge,
+  writeJson,
+  listDomains,
+  listEntities,
+  readEntityByRef,
+  resolveSource,
+  listFlagged,
+  summarizeEntities,
+};

package/skills/wix-replatform/resources/rp-target-wix/lib/ndjson.js

@@ -0,0 +1,260 @@
+'use strict';
+
+// rp-target-wix — NDJSON record-stream I/O for migration data files.
+//
+// Every migration writes the same shape of intermediate data: a long stream of per-entity
+// records that a later stage scans, batches and cursors through. That is a line-delimited
+// format, not a single JSON document, so it is built once here rather than re-derived by
+// codegen per project.
+//
+// WHY NDJSON over a JSON array (`{ entity, records: [...] }`):
+//   - scan: counting or filtering never parses the whole file, and never holds it in memory
+//   - batch: a bulk endpoint's page is `readBatches(file, 100)` — no slicing a giant array.
+//     Use `readBatchesByLimits` when the endpoint caps several dimensions at once (Wix bulk
+//     product create caps products, variants AND options per request).
+//   - cursor: resume from record N is a byte offset or a line skip, not a full re-parse
+//   - append: a producer can emit records as it finds them; a crash leaves a valid prefix
+//   - a single malformed record is one bad line, not an unparseable file
+//
+// USE IT FOR record streams (`data/source-extract/<entity>.ndjson`, crosswalks, audit logs).
+// Do NOT use it for single documents — a manifest, mapping plan, decisions or completion
+// report is one object and stays `.json`. Line-delimiting a document buys nothing and makes
+// it unreadable.
+//
+// Dependency-free and streaming, like the source adapters' transport modules. Codegen
+// vendors a copy into the project (e.g. `src/lib/ndjson.js`) and imports from it.
+
+const fs = require('node:fs');
+const fsp = require('node:fs/promises');
+const path = require('node:path');
+const readline = require('node:readline');
+
+const EXT = '.ndjson';
+
+function ensureDir(filePath) {
+  fs.mkdirSync(path.dirname(filePath), { recursive: true });
+}
+
+// One record per line. JSON.stringify escapes embedded newlines, so a record can never
+// break the line framing.
+function encodeRecord(record) {
+  if (record === undefined) throw new Error('ndjson: cannot encode undefined');
+  const line = JSON.stringify(record);
+  if (line === undefined) throw new Error('ndjson: record is not JSON-serialisable');
+  return `${line}\n`;
+}
+
+// --- writing ---------------------------------------------------------------
+
+// Streaming write of a whole stream. Accepts arrays, generators and async generators, so a
+// reader can hand over records as it produces them without materialising them all.
+async function writeRecords(filePath, records) {
+  ensureDir(filePath);
+  const out = fs.createWriteStream(filePath, { flags: 'w' });
+  let count = 0;
+  try {
+    for await (const record of records) {
+      count += 1;
+      if (!out.write(encodeRecord(record))) {
+        await new Promise((resolve) => out.once('drain', resolve));
+      }
+    }
+  } finally {
+    await new Promise((resolve, reject) => {
+      out.end((err) => (err ? reject(err) : resolve()));
+    });
+  }
+  return count;
+}
+
+// Incremental appender for producers that emit over time (crosswalk, audit log). `flush`
+// is per-record and O(1) — the whole point of not rewriting a JSON array each time.
+function createAppender(filePath) {
+  ensureDir(filePath);
+  let count = 0;
+  return {
+    file: filePath,
+    append(record) {
+      fs.appendFileSync(filePath, encodeRecord(record));
+      count += 1;
+      return record;
+    },
+    appendAll(records) {
+      let buffer = '';
+      for (const record of records) buffer += encodeRecord(record);
+      if (buffer) fs.appendFileSync(filePath, buffer);
+      count += records.length;
+      return records.length;
+    },
+    written: () => count,
+  };
+}
+
+// --- reading ---------------------------------------------------------------
+
+// Streams records without loading the file. readline handles \n, \r\n and the chunk-boundary
+// case that a naive split('\n') gets wrong.
+//
+// A blank line is skipped (a trailing newline is normal). A malformed line throws, naming the
+// line number — silently dropping records would corrupt a migration invisibly.
+async function* readRecords(filePath, { skipMalformed = false, onMalformed = null } = {}) {
+  if (!fs.existsSync(filePath)) return;
+  const input = fs.createReadStream(filePath, { encoding: 'utf8' });
+  const rl = readline.createInterface({ input, crlfDelay: Infinity });
+  let lineNumber = 0;
+  try {
+    for await (const line of rl) {
+      lineNumber += 1;
+      if (line.trim() === '') continue;
+      let record;
+      try {
+        record = JSON.parse(line);
+      } catch (err) {
+        const detail = `${filePath}:${lineNumber} is not valid JSON (${err.message})`;
+        if (onMalformed) onMalformed({ file: filePath, line: lineNumber, raw: line, error: err.message });
+        if (skipMalformed) continue;
+        throw new Error(`ndjson: ${detail}. Refusing to continue — a dropped record is invisible data loss.`);
+      }
+      yield record;
+    }
+  } finally {
+    rl.close();
+    input.destroy();
+  }
+}
+
+// The batching primitive bulk endpoints consume. `size` is a record count; when a target
+// caps something else too (Wix bulk product create allows 100 products AND 1000 variants
+// per request) use readBatchesBy instead.
+async function* readBatches(filePath, size = 100, options = {}) {
+  if (!Number.isInteger(size) || size < 1) throw new Error('ndjson: batch size must be a positive integer');
+  let batch = [];
+  for await (const record of readRecords(filePath, options)) {
+    batch.push(record);
+    if (batch.length >= size) {
+      yield batch;
+      batch = [];
+    }
+  }
+  if (batch.length) yield batch;
+}
+
+// Batch against SEVERAL simultaneous caps. Real bulk endpoints rarely have just one: Wix
+// bulk product create allows 100 products AND 1000 variants AND 100 options AND 100
+// modifiers AND 100 infoSections per request, and exceeding any single one rejects the whole
+// batch. Batching on record count alone silently produces requests that 428.
+//
+//   readBatchesByLimits(file, {
+//     limits: { records: 100, variants: 1000, options: 100 },
+//     cost: (p) => ({ variants: p.variants.length, options: p.options.length }),
+//   })
+//
+// `records` is implicit and always counted. Any dimension absent from `limits` is ignored,
+// so a cost function may report more than the caller caps.
+//
+// A single record that alone exceeds a cap is still emitted, in a batch of one, with
+// `oversized` reported through `onOversized`. Dropping it would be silent data loss and
+// splitting it is impossible — the caller has to decide (skip it, or fail loudly).
+async function* readBatchesByLimits(filePath, { limits = {}, cost = () => ({}), onOversized = null, ...options } = {}) {
+  const caps = { records: 100, ...limits };
+  for (const [dim, cap] of Object.entries(caps)) {
+    if (!(Number.isFinite(cap) && cap >= 1)) throw new Error(`ndjson: limit "${dim}" must be a number >= 1 (got ${cap})`);
+  }
+
+  let batch = [];
+  let running = {};
+  const costOf = (record) => {
+    const c = cost(record) || {};
+    return { ...c, records: 1 };
+  };
+  const wouldExceed = (c) => Object.keys(caps).some((dim) => (running[dim] || 0) + (c[dim] || 0) > caps[dim]);
+  const aloneExceeds = (c) => Object.keys(caps).some((dim) => (c[dim] || 0) > caps[dim]);
+
+  for await (const record of readRecords(filePath, options)) {
+    const c = costOf(record);
+    if (batch.length && wouldExceed(c)) {
+      yield batch;
+      batch = [];
+      running = {};
+    }
+    if (aloneExceeds(c) && onOversized) {
+      const over = Object.keys(caps).filter((dim) => (c[dim] || 0) > caps[dim]);
+      onOversized({ record, cost: c, exceeded: over, caps });
+    }
+    batch.push(record);
+    for (const dim of Object.keys(c)) running[dim] = (running[dim] || 0) + c[dim];
+  }
+  if (batch.length) yield batch;
+}
+
+// Single-dimension convenience wrapper over readBatchesByLimits, kept because plenty of
+// endpoints really do have just one cap.
+async function* readBatchesBy(filePath, { maxCount = 100, maxCost = Infinity, cost = () => 1, ...options } = {}) {
+  const limits = { records: maxCount };
+  if (Number.isFinite(maxCost)) limits.cost = maxCost;
+  yield* readBatchesByLimits(filePath, { ...options, limits, cost: (r) => ({ cost: cost(r) }) });
+}
+
+// Cursor: skip `offset` records, take at most `limit`. Used for resume — no need to parse
+// what has already been processed into objects.
+async function* readSlice(filePath, { offset = 0, limit = Infinity, ...options } = {}) {
+  let index = 0;
+  let taken = 0;
+  for await (const record of readRecords(filePath, options)) {
+    if (index++ < offset) continue;
+    if (taken++ >= limit) return;
+    yield record;
+  }
+}
+
+// Cheap count — parses nothing, just counts non-empty lines.
+async function countRecords(filePath) {
+  if (!fs.existsSync(filePath)) return 0;
+  const input = fs.createReadStream(filePath, { encoding: 'utf8' });
+  const rl = readline.createInterface({ input, crlfDelay: Infinity });
+  let count = 0;
+  try {
+    for await (const line of rl) if (line.trim() !== '') count += 1;
+  } finally {
+    rl.close();
+    input.destroy();
+  }
+  return count;
+}
+
+// Escape hatch for small streams that genuinely need to be in memory at once (a 6-record
+// category list). Named so that using it on a large stream is an obvious mistake.
+async function readAllRecords(filePath, options = {}) {
+  const out = [];
+  for await (const record of readRecords(filePath, options)) out.push(record);
+  return out;
+}
+
+// --- migration of existing projects ---------------------------------------
+
+// Converts a legacy `{ entity, recordCount, records: [...] }` document to NDJSON. Kept so a
+// project generated before this format change can be moved forward without re-extracting.
+async function convertLegacyJsonFile(jsonPath, ndjsonPath, { recordsKey = 'records' } = {}) {
+  const parsed = JSON.parse(await fsp.readFile(jsonPath, 'utf8'));
+  const records = Array.isArray(parsed) ? parsed : parsed[recordsKey];
+  if (!Array.isArray(records)) {
+    throw new Error(`ndjson: ${jsonPath} has no array at "${recordsKey}" to convert`);
+  }
+  const count = await writeRecords(ndjsonPath, records);
+  return { from: jsonPath, to: ndjsonPath, count };
+}
+
+module.exports = {
+  EXT,
+  encodeRecord,
+  writeRecords,
+  createAppender,
+  readRecords,
+  readBatches,
+  readBatchesBy,
+  readBatchesByLimits,
+  readSlice,
+  countRecords,
+  readAllRecords,
+  convertLegacyJsonFile,
+};