@objectstack/core 12.5.0 → 12.6.0

package/dist/index.d.cts

@@ -1862,6 +1862,83 @@ declare function calendarPartsInTz(d: Date, tz: string): CalendarParts;
  */
 declare function calendarPartsInTzOrUtc(d: Date, tz?: string): CalendarParts;
 
+/**
+ * `bulkWrite` — the shared batched-write helper used by BOTH the seed loader
+ * (`@objectstack/metadata-protocol`) and the data-import runner
+ * (`@objectstack/rest`), so neither reimplements batching, transient-error
+ * retry, or per-row degradation. See framework#2678.
+ *
+ * ObjectQL's engine already does the efficient thing when handed an ARRAY —
+ * one `driver.bulkCreate` round-trip plus parent-deduplicated summary
+ * recompute (`engine.insert(object, rows[])`) — but seed/import fed it one
+ * record at a time, so neither got the benefit. This module re-chunks rows
+ * into batches and drives them through a caller-supplied batch-write
+ * function, adding:
+ *
+ *  - transient-error retry (network blip / timeout) with exponential
+ *    backoff, so a dropped connection doesn't silently drop the row (the
+ *    2026-07-06 HotCRM incident: a turso `fetch failed` mid-seed dropped rows
+ *    silently because nothing retried);
+ *  - per-row degradation when a batch fails for a non-transient (logical /
+ *    validation) reason, so one bad row can't fail the other N-1 — needed
+ *    because `driver.bulkCreate` is a single multi-row statement/`Promise.all`
+ *    on every driver in this repo (sql, memory, mongodb): one bad row fails
+ *    the whole call;
+ *  - a stable per-row result keyed by the row's original index, so callers
+ *    can reassemble output in input order even though rows are processed in
+ *    batches (and a batch's flush may be interleaved with other, immediate,
+ *    per-row work such as updates).
+ */
+interface BulkWriteRowResult<TRecord = any> {
+    /** Index into the original `rows` array passed to {@link bulkWrite}. */
+    index: number;
+    ok: boolean;
+    record?: TRecord;
+    error?: unknown;
+}
+interface RetryOptions {
+    /** Max attempts for one write (batch or single-row), including the first. Default 3. */
+    maxRetries?: number;
+    /** Base backoff in ms; doubled each retry, plus jitter. Default 200. */
+    backoffBaseMs?: number;
+    /** Classifies an error as transient (worth retrying) vs logical (the row/batch is just bad). */
+    isTransientError?: (err: unknown) => boolean;
+    /** Injectable sleep, for deterministic tests. */
+    sleep?: (ms: number) => Promise<void>;
+}
+interface BulkWriteOptions<TRow, TRecord = any> extends RetryOptions {
+    /** Rows per batch. Default 200 (framework#2678 suggests 100-500). */
+    batchSize?: number;
+    /**
+     * Write one batch. MUST resolve to one record per input row, in the SAME
+     * order as `batch` — {@link bulkWrite} correlates `records[i]` back to
+     * `batch[i]` positionally (this is how every `bulkCreate` implementation in
+     * this repo already behaves: sql's single `INSERT ... VALUES (...), (...)
+     * RETURNING *`, memory's `Promise.all`, mongodb's ordered `insertMany`).
+     */
+    writeBatch: (batch: TRow[]) => Promise<TRecord[]>;
+    /** Write a single row — used only to degrade a failed batch. */
+    writeOne: (row: TRow) => Promise<TRecord>;
+}
+declare function defaultIsTransientError(err: unknown): boolean;
+/**
+ * Retry a single write (e.g. an `engine.update()` call the seed loader or
+ * import runner makes outside the batched-insert path) with the same
+ * transient-error backoff {@link bulkWrite} applies to batches — so a
+ * network blip doesn't drop an update the way it used to drop an insert.
+ */
+declare function withTransientRetry<T>(fn: () => Promise<T>, opts?: RetryOptions): Promise<T>;
+/**
+ * Write `rows` through `opts.writeBatch` in chunks of `opts.batchSize`,
+ * retrying a whole-batch transient failure with backoff, and degrading to
+ * per-row `opts.writeOne` calls (each itself retried) when a batch fails for
+ * a non-transient reason — so one bad row can't drop the rest of the batch.
+ *
+ * Returns one {@link BulkWriteRowResult} per input row, indexed to match
+ * `rows`' original order.
+ */
+declare function bulkWrite<TRow, TRecord = any>(rows: TRow[], opts: BulkWriteOptions<TRow, TRecord>): Promise<BulkWriteRowResult<TRecord>[]>;
+
 /**
  * In-memory Map-backed cache fallback.
  *
@@ -2298,4 +2375,4 @@ declare class NamespaceResolver {
     private suggestAlternative;
 }
 
-export { API_KEY_PREFIX, type ApiKeyPrincipal, ApiRegistry, type ApiRegistryPluginConfig, type AuthGate, CORE_FALLBACK_FACTORIES, type CalendarParts, DependencyResolver, type GeneratedApiKey, HotReloadManager, type KernelState, type KeyInput, LiteKernel, type NamespaceCheckResult, type NamespaceConflict, type NamespaceEntry, NamespaceResolver, ObjectKernel, ObjectKernelBase, type ObjectKernelConfig, ObjectLogger, type ParsedSignature, type PermissionCheckResult$1 as PermissionCheckResult, type PermissionGrant, type Plugin, type PluginArtifactVerifyResult, PluginConfigValidator, type PluginContext, PluginHealthMonitor, type PluginHealthStatus, type PluginLoadResult, PluginLoader, type PluginMetadata, type PermissionCheckResult as PluginPermissionCheckResult, PluginPermissionEnforcer, PluginPermissionManager, type PluginPermissions, PluginSandboxRuntime, PluginSecurityScanner, type PluginSignatureConfig, PluginSignatureVerifier, type PluginStartupResult, type PublisherVerifyResult, index as QA, type ResolveAuthzInput, type ResolveLocalizationInput, type ResolvedAuthzContext, type ResourceUsage, SIGNATURE_ALG, type SandboxContext, type ScanTarget, SecurePluginContext, type SecurityIssue, SemanticVersionManager, type ServiceFactory, ServiceLifecycle, type ServiceRegistration, type SignatureVerificationResult, type VersionCompatibility, buildPermissionsFromGrants, calendarPartsInTz, calendarPartsInTzOrUtc, counterSignPayload, createApiRegistryPlugin, createMemoryCache, createMemoryI18n, createMemoryJob, createMemoryMetadata, createMemoryQueue, createPluginConfigValidator, createPluginPermissionEnforcer, deepMerge, evaluateAuthGate, extractApiKey, generateApiKey, generateEd25519KeyPair, getEnv, getMemoryUsage, hashApiKey, isAuthGateAllowlisted, isExpired, isNode, parseScopes, parseSignature, readAuthoredTranslationLayer, resolveApiKeyPrincipal, resolveAuthzContext, resolveLocale, resolveLocalizationContext, safeExit, signPayload, verifyPayload, verifyPlatformSignature, verifyPluginArtifact, verifyPublisherSignature, wireAuthoredTranslationSync };
+export { API_KEY_PREFIX, type ApiKeyPrincipal, ApiRegistry, type ApiRegistryPluginConfig, type AuthGate, type BulkWriteOptions, type BulkWriteRowResult, CORE_FALLBACK_FACTORIES, type CalendarParts, DependencyResolver, type GeneratedApiKey, HotReloadManager, type KernelState, type KeyInput, LiteKernel, type NamespaceCheckResult, type NamespaceConflict, type NamespaceEntry, NamespaceResolver, ObjectKernel, ObjectKernelBase, type ObjectKernelConfig, ObjectLogger, type ParsedSignature, type PermissionCheckResult$1 as PermissionCheckResult, type PermissionGrant, type Plugin, type PluginArtifactVerifyResult, PluginConfigValidator, type PluginContext, PluginHealthMonitor, type PluginHealthStatus, type PluginLoadResult, PluginLoader, type PluginMetadata, type PermissionCheckResult as PluginPermissionCheckResult, PluginPermissionEnforcer, PluginPermissionManager, type PluginPermissions, PluginSandboxRuntime, PluginSecurityScanner, type PluginSignatureConfig, PluginSignatureVerifier, type PluginStartupResult, type PublisherVerifyResult, index as QA, type ResolveAuthzInput, type ResolveLocalizationInput, type ResolvedAuthzContext, type ResourceUsage, type RetryOptions, SIGNATURE_ALG, type SandboxContext, type ScanTarget, SecurePluginContext, type SecurityIssue, SemanticVersionManager, type ServiceFactory, ServiceLifecycle, type ServiceRegistration, type SignatureVerificationResult, type VersionCompatibility, buildPermissionsFromGrants, bulkWrite, calendarPartsInTz, calendarPartsInTzOrUtc, counterSignPayload, createApiRegistryPlugin, createMemoryCache, createMemoryI18n, createMemoryJob, createMemoryMetadata, createMemoryQueue, createPluginConfigValidator, createPluginPermissionEnforcer, deepMerge, defaultIsTransientError, evaluateAuthGate, extractApiKey, generateApiKey, generateEd25519KeyPair, getEnv, getMemoryUsage, hashApiKey, isAuthGateAllowlisted, isExpired, isNode, parseScopes, parseSignature, readAuthoredTranslationLayer, resolveApiKeyPrincipal, resolveAuthzContext, resolveLocale, resolveLocalizationContext, safeExit, signPayload, verifyPayload, verifyPlatformSignature, verifyPluginArtifact, verifyPublisherSignature, wireAuthoredTranslationSync, withTransientRetry };

package/dist/index.d.ts

@@ -1862,6 +1862,83 @@ declare function calendarPartsInTz(d: Date, tz: string): CalendarParts;
  */
 declare function calendarPartsInTzOrUtc(d: Date, tz?: string): CalendarParts;
 
+/**
+ * `bulkWrite` — the shared batched-write helper used by BOTH the seed loader
+ * (`@objectstack/metadata-protocol`) and the data-import runner
+ * (`@objectstack/rest`), so neither reimplements batching, transient-error
+ * retry, or per-row degradation. See framework#2678.
+ *
+ * ObjectQL's engine already does the efficient thing when handed an ARRAY —
+ * one `driver.bulkCreate` round-trip plus parent-deduplicated summary
+ * recompute (`engine.insert(object, rows[])`) — but seed/import fed it one
+ * record at a time, so neither got the benefit. This module re-chunks rows
+ * into batches and drives them through a caller-supplied batch-write
+ * function, adding:
+ *
+ *  - transient-error retry (network blip / timeout) with exponential
+ *    backoff, so a dropped connection doesn't silently drop the row (the
+ *    2026-07-06 HotCRM incident: a turso `fetch failed` mid-seed dropped rows
+ *    silently because nothing retried);
+ *  - per-row degradation when a batch fails for a non-transient (logical /
+ *    validation) reason, so one bad row can't fail the other N-1 — needed
+ *    because `driver.bulkCreate` is a single multi-row statement/`Promise.all`
+ *    on every driver in this repo (sql, memory, mongodb): one bad row fails
+ *    the whole call;
+ *  - a stable per-row result keyed by the row's original index, so callers
+ *    can reassemble output in input order even though rows are processed in
+ *    batches (and a batch's flush may be interleaved with other, immediate,
+ *    per-row work such as updates).
+ */
+interface BulkWriteRowResult<TRecord = any> {
+    /** Index into the original `rows` array passed to {@link bulkWrite}. */
+    index: number;
+    ok: boolean;
+    record?: TRecord;
+    error?: unknown;
+}
+interface RetryOptions {
+    /** Max attempts for one write (batch or single-row), including the first. Default 3. */
+    maxRetries?: number;
+    /** Base backoff in ms; doubled each retry, plus jitter. Default 200. */
+    backoffBaseMs?: number;
+    /** Classifies an error as transient (worth retrying) vs logical (the row/batch is just bad). */
+    isTransientError?: (err: unknown) => boolean;
+    /** Injectable sleep, for deterministic tests. */
+    sleep?: (ms: number) => Promise<void>;
+}
+interface BulkWriteOptions<TRow, TRecord = any> extends RetryOptions {
+    /** Rows per batch. Default 200 (framework#2678 suggests 100-500). */
+    batchSize?: number;
+    /**
+     * Write one batch. MUST resolve to one record per input row, in the SAME
+     * order as `batch` — {@link bulkWrite} correlates `records[i]` back to
+     * `batch[i]` positionally (this is how every `bulkCreate` implementation in
+     * this repo already behaves: sql's single `INSERT ... VALUES (...), (...)
+     * RETURNING *`, memory's `Promise.all`, mongodb's ordered `insertMany`).
+     */
+    writeBatch: (batch: TRow[]) => Promise<TRecord[]>;
+    /** Write a single row — used only to degrade a failed batch. */
+    writeOne: (row: TRow) => Promise<TRecord>;
+}
+declare function defaultIsTransientError(err: unknown): boolean;
+/**
+ * Retry a single write (e.g. an `engine.update()` call the seed loader or
+ * import runner makes outside the batched-insert path) with the same
+ * transient-error backoff {@link bulkWrite} applies to batches — so a
+ * network blip doesn't drop an update the way it used to drop an insert.
+ */
+declare function withTransientRetry<T>(fn: () => Promise<T>, opts?: RetryOptions): Promise<T>;
+/**
+ * Write `rows` through `opts.writeBatch` in chunks of `opts.batchSize`,
+ * retrying a whole-batch transient failure with backoff, and degrading to
+ * per-row `opts.writeOne` calls (each itself retried) when a batch fails for
+ * a non-transient reason — so one bad row can't drop the rest of the batch.
+ *
+ * Returns one {@link BulkWriteRowResult} per input row, indexed to match
+ * `rows`' original order.
+ */
+declare function bulkWrite<TRow, TRecord = any>(rows: TRow[], opts: BulkWriteOptions<TRow, TRecord>): Promise<BulkWriteRowResult<TRecord>[]>;
+
 /**
  * In-memory Map-backed cache fallback.
  *
@@ -2298,4 +2375,4 @@ declare class NamespaceResolver {
     private suggestAlternative;
 }
 
-export { API_KEY_PREFIX, type ApiKeyPrincipal, ApiRegistry, type ApiRegistryPluginConfig, type AuthGate, CORE_FALLBACK_FACTORIES, type CalendarParts, DependencyResolver, type GeneratedApiKey, HotReloadManager, type KernelState, type KeyInput, LiteKernel, type NamespaceCheckResult, type NamespaceConflict, type NamespaceEntry, NamespaceResolver, ObjectKernel, ObjectKernelBase, type ObjectKernelConfig, ObjectLogger, type ParsedSignature, type PermissionCheckResult$1 as PermissionCheckResult, type PermissionGrant, type Plugin, type PluginArtifactVerifyResult, PluginConfigValidator, type PluginContext, PluginHealthMonitor, type PluginHealthStatus, type PluginLoadResult, PluginLoader, type PluginMetadata, type PermissionCheckResult as PluginPermissionCheckResult, PluginPermissionEnforcer, PluginPermissionManager, type PluginPermissions, PluginSandboxRuntime, PluginSecurityScanner, type PluginSignatureConfig, PluginSignatureVerifier, type PluginStartupResult, type PublisherVerifyResult, index as QA, type ResolveAuthzInput, type ResolveLocalizationInput, type ResolvedAuthzContext, type ResourceUsage, SIGNATURE_ALG, type SandboxContext, type ScanTarget, SecurePluginContext, type SecurityIssue, SemanticVersionManager, type ServiceFactory, ServiceLifecycle, type ServiceRegistration, type SignatureVerificationResult, type VersionCompatibility, buildPermissionsFromGrants, calendarPartsInTz, calendarPartsInTzOrUtc, counterSignPayload, createApiRegistryPlugin, createMemoryCache, createMemoryI18n, createMemoryJob, createMemoryMetadata, createMemoryQueue, createPluginConfigValidator, createPluginPermissionEnforcer, deepMerge, evaluateAuthGate, extractApiKey, generateApiKey, generateEd25519KeyPair, getEnv, getMemoryUsage, hashApiKey, isAuthGateAllowlisted, isExpired, isNode, parseScopes, parseSignature, readAuthoredTranslationLayer, resolveApiKeyPrincipal, resolveAuthzContext, resolveLocale, resolveLocalizationContext, safeExit, signPayload, verifyPayload, verifyPlatformSignature, verifyPluginArtifact, verifyPublisherSignature, wireAuthoredTranslationSync };
+export { API_KEY_PREFIX, type ApiKeyPrincipal, ApiRegistry, type ApiRegistryPluginConfig, type AuthGate, type BulkWriteOptions, type BulkWriteRowResult, CORE_FALLBACK_FACTORIES, type CalendarParts, DependencyResolver, type GeneratedApiKey, HotReloadManager, type KernelState, type KeyInput, LiteKernel, type NamespaceCheckResult, type NamespaceConflict, type NamespaceEntry, NamespaceResolver, ObjectKernel, ObjectKernelBase, type ObjectKernelConfig, ObjectLogger, type ParsedSignature, type PermissionCheckResult$1 as PermissionCheckResult, type PermissionGrant, type Plugin, type PluginArtifactVerifyResult, PluginConfigValidator, type PluginContext, PluginHealthMonitor, type PluginHealthStatus, type PluginLoadResult, PluginLoader, type PluginMetadata, type PermissionCheckResult as PluginPermissionCheckResult, PluginPermissionEnforcer, PluginPermissionManager, type PluginPermissions, PluginSandboxRuntime, PluginSecurityScanner, type PluginSignatureConfig, PluginSignatureVerifier, type PluginStartupResult, type PublisherVerifyResult, index as QA, type ResolveAuthzInput, type ResolveLocalizationInput, type ResolvedAuthzContext, type ResourceUsage, type RetryOptions, SIGNATURE_ALG, type SandboxContext, type ScanTarget, SecurePluginContext, type SecurityIssue, SemanticVersionManager, type ServiceFactory, ServiceLifecycle, type ServiceRegistration, type SignatureVerificationResult, type VersionCompatibility, buildPermissionsFromGrants, bulkWrite, calendarPartsInTz, calendarPartsInTzOrUtc, counterSignPayload, createApiRegistryPlugin, createMemoryCache, createMemoryI18n, createMemoryJob, createMemoryMetadata, createMemoryQueue, createPluginConfigValidator, createPluginPermissionEnforcer, deepMerge, defaultIsTransientError, evaluateAuthGate, extractApiKey, generateApiKey, generateEd25519KeyPair, getEnv, getMemoryUsage, hashApiKey, isAuthGateAllowlisted, isExpired, isNode, parseScopes, parseSignature, readAuthoredTranslationLayer, resolveApiKeyPrincipal, resolveAuthzContext, resolveLocale, resolveLocalizationContext, safeExit, signPayload, verifyPayload, verifyPlatformSignature, verifyPluginArtifact, verifyPublisherSignature, wireAuthoredTranslationSync, withTransientRetry };

package/dist/index.js

@@ -4389,6 +4389,87 @@ function calendarPartsInTzOrUtc(d, tz) {
   };
 }
 
+// src/utils/bulk-write.ts
+var DEFAULT_BATCH_SIZE = 200;
+var DEFAULT_MAX_RETRIES = 3;
+var DEFAULT_BACKOFF_BASE_MS = 200;
+var TRANSIENT_PATTERNS = [
+  /fetch failed/i,
+  /network/i,
+  /timed?\s*out/i,
+  /timeout/i,
+  /socket hang ?up/i,
+  /connection.*(closed|reset|refused|terminated|aborted)/i,
+  /\b(502|503|504)\b/,
+  /server.*unavailable/i,
+  /too many connections/i
+];
+var TRANSIENT_CODES = /^(ECONNRESET|ECONNREFUSED|ECONNABORTED|EPIPE|EAI_AGAIN|ETIMEDOUT|EHOSTUNREACH|ENETUNREACH|ENOTFOUND)$/i;
+function defaultIsTransientError(err) {
+  const code = err?.code;
+  if (typeof code === "string" && TRANSIENT_CODES.test(code)) return true;
+  const message = err?.message;
+  const text = typeof message === "string" ? message : String(err ?? "");
+  return TRANSIENT_PATTERNS.some((re) => re.test(text));
+}
+var defaultSleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
+async function withRetry(fn, opts) {
+  let lastError;
+  for (let attempt = 1; attempt <= opts.maxRetries; attempt++) {
+    try {
+      return await fn();
+    } catch (err) {
+      lastError = err;
+      if (attempt >= opts.maxRetries || !opts.isTransientError(err)) throw err;
+      const jitter = Math.floor(Math.random() * 50);
+      await opts.sleep(opts.backoffBaseMs * 2 ** (attempt - 1) + jitter);
+    }
+  }
+  throw lastError;
+}
+async function withTransientRetry(fn, opts = {}) {
+  return withRetry(fn, {
+    maxRetries: Math.max(1, opts.maxRetries ?? DEFAULT_MAX_RETRIES),
+    backoffBaseMs: opts.backoffBaseMs ?? DEFAULT_BACKOFF_BASE_MS,
+    isTransientError: opts.isTransientError ?? defaultIsTransientError,
+    sleep: opts.sleep ?? defaultSleep
+  });
+}
+async function bulkWrite(rows, opts) {
+  const batchSize = Math.max(1, opts.batchSize ?? DEFAULT_BATCH_SIZE);
+  const retryOpts = {
+    maxRetries: Math.max(1, opts.maxRetries ?? DEFAULT_MAX_RETRIES),
+    backoffBaseMs: opts.backoffBaseMs ?? DEFAULT_BACKOFF_BASE_MS,
+    isTransientError: opts.isTransientError ?? defaultIsTransientError,
+    sleep: opts.sleep ?? defaultSleep
+  };
+  const results = new Array(rows.length);
+  for (let start = 0; start < rows.length; start += batchSize) {
+    const batch = rows.slice(start, start + batchSize);
+    try {
+      const records = await withRetry(() => opts.writeBatch(batch), retryOpts);
+      for (let i = 0; i < batch.length; i++) {
+        results[start + i] = { index: start + i, ok: true, record: records[i] };
+      }
+    } catch (batchErr) {
+      if (batch.length === 1) {
+        results[start] = { index: start, ok: false, error: batchErr };
+        continue;
+      }
+      for (let i = 0; i < batch.length; i++) {
+        const idx = start + i;
+        try {
+          const record = await withRetry(() => opts.writeOne(batch[i]), retryOpts);
+          results[idx] = { index: idx, ok: true, record };
+        } catch (err) {
+          results[idx] = { index: idx, ok: false, error: err };
+        }
+      }
+    }
+  }
+  return results;
+}
+
 // src/health-monitor.ts
 var PluginHealthMonitor = class {
   constructor(logger) {
@@ -5327,6 +5408,7 @@ export {
   SemanticVersionManager,
   ServiceLifecycle,
   buildPermissionsFromGrants,
+  bulkWrite,
   calendarPartsInTz,
   calendarPartsInTzOrUtc,
   counterSignPayload,
@@ -5340,6 +5422,7 @@ export {
   createPluginConfigValidator,
   createPluginPermissionEnforcer,
   deepMerge,
+  defaultIsTransientError,
   evaluateAuthGate,
   extractApiKey,
   generateApiKey,
@@ -5363,6 +5446,7 @@ export {
   verifyPlatformSignature,
   verifyPluginArtifact,
   verifyPublisherSignature,
-  wireAuthoredTranslationSync
+  wireAuthoredTranslationSync,
+  withTransientRetry
 };
 //# sourceMappingURL=index.js.map