@gentleduck/upload 0.2.1

package/src/core/engine/store/store.libs.ts

@@ -0,0 +1,169 @@
+import type { UploadConfig } from '../../client'
+import type { AnyCursor, CursorMap, IntentMap, UploadError, UploadResultBase } from '../../contracts'
+import { DEFAULT_RETRY_DELAY_BASE_MS, DEFAULT_RETRY_DELAY_MAX_MS } from '../../utils/constants'
+import { computeFingerprint, fingerprintMatches } from '../../utils/fingerprint'
+import { isRecord } from '../../utils/guards'
+import type { UploadItem } from '../internal-events.types'
+
+export { sleep } from '../../utils/async'
+
+import type { StoreOptions } from './store.types'
+
+/**
+ * Calculates SHA256 checksum of a file for deduplication.
+ * Uses Web Crypto API for efficient hashing.
+ */
+export async function calculateFileChecksum(file: File): Promise<string> {
+  const buffer = await file.arrayBuffer()
+  const hashBuffer = await crypto.subtle.digest('SHA-256', buffer)
+  const hashArray = Array.from(new Uint8Array(hashBuffer))
+  return hashArray.map((b) => b.toString(16).padStart(2, '0')).join('')
+}
+
+/**
+ * Determines whether items for a given purpose should auto-start.
+ *
+ * Supports three configuration shapes:
+ * - `false | undefined`: never auto-start
+ * - `P[]`: auto-start for purposes included in the list
+ * - `(purpose: P) => boolean`: custom predicate
+ */
+export function isAutoStart<M extends IntentMap, C extends CursorMap<M>, P extends string, R extends UploadResultBase>(
+  opts: StoreOptions<M, C, P, R>,
+  purpose: P,
+): boolean {
+  const v = opts.config.autoStart
+  if (v === undefined) return false
+  if (Array.isArray(v)) return v.includes(purpose)
+  if (typeof v === 'function') return v(purpose)
+  return false
+}
+
+/**
+ * Builds a lightweight fingerprint from a {@link File}.
+ *
+ * This fingerprint is used for display and basic identity. If you need stronger identity
+ * (e.g. SHA-256), provide a custom {@link StoreOptions.fingerprint} function or let the
+ * add-file handler compute a checksum asynchronously.
+ */
+export { computeFingerprint, fingerprintMatches }
+
+/**
+ * Normalizes unknown thrown values into the engine's {@link UploadError} shape.
+ *
+ * You can provide a custom normalizer through {@link StoreOptions.errorNormalizer}.
+ * Built-in behavior:
+ * - detects abort errors and marks them non-retryable
+ * - classifies common network and HTTP errors as retryable where appropriate
+ */
+export function normalizeError(err: unknown, customNormalizer?: (err: unknown) => UploadError): UploadError {
+  if (customNormalizer) return customNormalizer(err)
+
+  // Transport abort errors can carry {code:'aborted', reason:'pause'|'cancel'}
+  if (isAbortError(err)) {
+    return { code: 'aborted', message: 'Upload aborted', reason: String(err.reason ?? 'unknown'), retryable: false }
+  }
+
+  const msg = isRecord(err) && typeof err.message === 'string' ? err.message : 'Unknown error'
+
+  // Network errors
+  if (String(msg).toLowerCase().includes('network') || String(msg).toLowerCase().includes('fetch')) {
+    return { code: 'network', message: String(msg), cause: err, retryable: true }
+  }
+
+  // HTTP errors
+  if (isRecord(err) && (typeof err.status === 'number' || typeof err.statusCode === 'number')) {
+    const status = typeof err.status === 'number' ? err.status : err.statusCode
+    const retryable = status >= 500 || status === 429
+    return { code: 'http', status, message: String(msg), cause: err, retryable }
+  }
+
+  return { code: 'unknown', message: String(msg), cause: err, retryable: false }
+}
+
+/**
+ * Type guard for items that currently hold a concrete backend intent.
+ */
+export function hasIntent<M extends IntentMap, C extends CursorMap<M>, P extends string, R extends UploadResultBase>(
+  item: UploadItem<M, C, P, R>,
+): item is Extract<UploadItem<M, C, P, R>, { intent: M[keyof M] }> {
+  return 'intent' in item && !!item.intent
+}
+
+/**
+ * Type guard for items that currently have a bound {@link File}.
+ *
+ * Persisted items restored from storage may lack `file` until the UI re-binds it.
+ */
+export function hasFile<M extends IntentMap, C extends CursorMap<M>, P extends string, R extends UploadResultBase>(
+  item: UploadItem<M, C, P, R>,
+): item is Extract<UploadItem<M, C, P, R>, { file: File }> {
+  return 'file' in item && !!item.file
+}
+
+/**
+ * Type guard that narrows to item variants carrying a cursor field.
+ *
+ * Cursor presence depends on the current phase and the strategy.
+ */
+export function hasCursor<M extends IntentMap, C extends CursorMap<M>, P extends string, R extends UploadResultBase>(
+  item: UploadItem<M, C, P, R>,
+): item is Extract<UploadItem<M, C, P, R>, { cursor?: AnyCursor<C> }> {
+  return 'cursor' in item
+}
+
+/**
+ * Runtime guard for multipart intent objects.
+ *
+ * Used for best-effort abort calls when canceling multipart uploads.
+ */
+export function isMultipartIntent(
+  intent: unknown,
+): intent is { strategy: 'multipart'; fileId: string; uploadId: string; partSize: number } {
+  return (
+    isRecord(intent) &&
+    intent.strategy === 'multipart' &&
+    typeof intent.fileId === 'string' &&
+    typeof intent.uploadId === 'string'
+  )
+}
+
+/**
+ * Runtime guard for abort errors produced by the transport/strategy layer.
+ */
+export function isAbortError(err: unknown): err is { code: 'aborted'; reason?: unknown } {
+  return isRecord(err) && err.code === 'aborted'
+}
+
+/**
+ * Small helper to narrow unknown values to objects.
+ */
+
+/**
+ * Computes whether a failed operation should be retried and, if so, after what delay.
+ *
+ * If {@link UploadConfig.retryPolicy} is provided it is used as the source of truth.
+ * Otherwise the default policy:
+ * - never retries auth/validation/strategy_missing/aborted
+ * - retries up to `maxAttempts` with exponential backoff and a fixed cap
+ */
+export function retryDecision<P extends string>(
+  config: UploadConfig<P>,
+  ctx: { phase: 'intent' | 'upload' | 'complete'; attempt: number; error: UploadError },
+) {
+  if (config.retryPolicy) return config.retryPolicy(ctx)
+
+  // Default: retry network/server unknown errors, not auth/validation/strategy_missing
+  if (ctx.error.code === 'auth') return { retryable: false }
+  if (ctx.error.code === 'validation_failed') return { retryable: false }
+  if (ctx.error.code === 'strategy_missing') return { retryable: false }
+  if (ctx.error.code === 'aborted') return { retryable: false }
+
+  const maxAttempts = config.maxAttempts
+  const retryable = ctx.attempt < maxAttempts
+  if (!retryable) return { retryable: false }
+
+  // Exponential backoff with cap
+  const delayMs = Math.min(DEFAULT_RETRY_DELAY_MAX_MS, DEFAULT_RETRY_DELAY_BASE_MS * 2 ** (ctx.attempt - 1))
+  return { retryable: true, delayMs }
+}

package/src/core/engine/store/store.runtime.ts

@@ -0,0 +1,336 @@
+import { resolveUploadConfig } from '../../client'
+import type { CursorMap, IntentMap, UploadResultBase } from '../../contracts'
+import { createXHRTransport } from '../../contracts/transport'
+import { deserializeSnapshot, serializeSnapshot } from '../../persistence'
+import { createTypedEmitter } from '../../utils/emitter'
+import type { UploadCommand } from '../commands.types'
+import type { UploadEventMap } from '../event-map.types'
+import type { InternalEvent, UploadItem } from '../internal-events.types'
+import { createReducer, type UploadState } from '../reducer'
+import { cleanupOldItems } from './handlers/clean-up'
+import { scheduleWork } from './store.schedule'
+import type { InflightUpload, StoreOptions, StoreRuntime } from './store.types'
+
+/**
+ * Creates a new runtime instance with initial state, emitter, reducer, and effect queue.
+ *
+ * Notes:
+ * - Reducers remain pure: all async work is enqueued through {@link StoreRuntime.enqueueEffect}.
+ * - {@link StoreRuntime.applyInternal} invokes hooks, performs cleanup, and then schedules more work.
+ * - {@link StoreRuntime.dispatch} is intentionally assigned later to break import cycles.
+ *
+ * @template M - Intent map type
+ * @template C - Cursor map type
+ * @template P - Purpose string union type
+ *
+ * @param opts - Construction options for the store.
+ */
+export function createStoreRuntime<
+  M extends IntentMap,
+  C extends CursorMap<M>,
+  P extends string,
+  R extends UploadResultBase = UploadResultBase,
+>(opts: StoreOptions<M, C, P, R>): StoreRuntime<M, C, P, R> {
+  const resolvedOpts = resolveStoreOptions(opts)
+  const persistence = resolvedOpts.persistence
+
+  // Debounced persistence flush. Persistence writes a single namespace `key`,
+  // and the adapter can choose how to store multiple item records under it.
+  let persistTimer: ReturnType<typeof setTimeout> | null = null
+  const persistDebounceMs = Math.max(0, persistence?.debounceMs ?? 200)
+
+  const flushPersistence = async () => {
+    if (!persistence) return
+    try {
+      const snap = (persistence.serialize ?? serializeSnapshot)(rt.state, persistence.version)
+
+      // If there's nothing to persist, clear the namespace to keep storage clean.
+      if (!snap.items || Object.keys(snap.items).length === 0) {
+        await persistence.adapter.clear(persistence.key)
+        return
+      }
+
+      await persistence.adapter.save(persistence.key, snap)
+    } catch (err) {
+      if (typeof window !== 'undefined' && process.env.NODE_ENV === 'development') {
+        console.warn('[UploadEngine] persistence flush failed:', err)
+      }
+    }
+  }
+
+  const schedulePersistence = () => {
+    if (!persistence) return
+    if (persistDebounceMs === 0) {
+      void flushPersistence()
+      return
+    }
+    if (persistTimer) clearTimeout(persistTimer)
+    persistTimer = setTimeout(() => void flushPersistence(), persistDebounceMs)
+  }
+
+  const initialState: UploadState<M, C, P, R> = resolvedOpts.initialState ?? {
+    items: new Map<string, UploadItem<M, C, P, R>>(),
+  }
+
+  const effectQueue: Array<() => Promise<void>> = []
+
+  const rt: StoreRuntime<M, C, P, R> = {
+    opts: resolvedOpts,
+    state: initialState,
+
+    listeners: new Set<() => void>(),
+    emitter: createTypedEmitter<UploadEventMap<M, C, P, R>>(),
+    reduce: createReducer<M, C, P, R>(),
+
+    inflightUploads: new Map<string, InflightUpload>(),
+    inflightIntents: new Map<string, AbortController>(),
+    inflightCompletes: new Map<string, AbortController>(),
+
+    effectQueue,
+    processingEffects: false,
+    scheduling: false,
+
+    notify() {
+      rt.listeners.forEach((l) => l())
+    },
+
+    scheduleWork() {
+      scheduleWork(rt)
+    },
+
+    applyInternal(event: InternalEvent<M, C, P, R>) {
+      const prev = rt.state
+      const next = rt.reduce(prev, event)
+      rt.state = next
+      rt.opts.hooks?.onInternalEvent?.(event, rt.state)
+
+      emitInternalEvent(rt, event)
+
+      const cleaned = cleanupOldItems(rt.opts, rt.state)
+      if (cleaned) rt.state = cleaned
+
+      schedulePersistence()
+      rt.notify()
+      rt.scheduleWork()
+    },
+
+    applyCommand(cmd: UploadCommand<P>) {
+      const prev = rt.state
+      const next = rt.reduce(prev, cmd)
+      rt.state = next
+
+      emitCommandEvents(rt, cmd, prev, next)
+
+      const cleaned = cleanupOldItems(rt.opts, rt.state)
+      if (cleaned) rt.state = cleaned
+
+      schedulePersistence()
+      rt.notify()
+      rt.scheduleWork()
+    },
+
+    enqueueEffect(effect: () => Promise<void>) {
+      rt.effectQueue.push(effect)
+      if (!rt.processingEffects) void rt.processEffects()
+    },
+
+    async processEffects() {
+      rt.processingEffects = true
+      while (rt.effectQueue.length > 0) {
+        const effect = rt.effectQueue.shift()
+        if (!effect) continue
+        try {
+          await effect()
+        } catch (err) {
+          if (typeof window !== 'undefined' && process.env.NODE_ENV === 'development') {
+            const error = err instanceof Error ? err : new Error(String(err))
+            const context = {
+              queueLength: rt.effectQueue.length,
+              processing: rt.processingEffects,
+              timestamp: new Date().toISOString(),
+            }
+            console.error('[UploadEngine] Effect error:', error, context)
+          }
+        }
+      }
+      rt.processingEffects = false
+    },
+
+    // replaced by createUploadStore
+    dispatch: () => {
+      throw new Error('[UploadEngine] dispatch not initialized')
+    },
+  } satisfies StoreRuntime<M, C, P, R>
+
+  // Hydrate from persistence when no initialState was provided.
+  // For async adapters (IndexedDB), this happens after construction.
+  if (persistence && !resolvedOpts.initialState && typeof window !== 'undefined') {
+    try {
+      const loaded = persistence.adapter.load(persistence.key)
+      const applyLoaded = (raw: unknown | null) => {
+        if (!raw) return
+        const next = (persistence.deserialize ?? deserializeSnapshot)(raw, {
+          isPurpose: persistence.isPurpose,
+          isIntent: persistence.isIntent,
+          hasStrategy: resolvedOpts.strategies.has,
+        })
+        if (!next) return
+
+        // Only merge into an empty store to avoid clobbering user actions
+        // that happened before the async load completed.
+        if (rt.state.items.size === 0) {
+          rt.state = next
+        } else {
+          const merged = new Map(rt.state.items)
+          for (const [id, item] of next.items.entries()) {
+            if (!merged.has(id)) merged.set(id, item)
+          }
+          rt.state = { ...rt.state, items: merged }
+        }
+
+        rt.notify()
+        rt.scheduleWork()
+      }
+
+      void Promise.resolve(loaded)
+        .then(applyLoaded)
+        .catch(() => undefined)
+    } catch {
+      // ignore
+    }
+  }
+
+  return rt
+}
+
+function resolveStoreOptions<M extends IntentMap, C extends CursorMap<M>, P extends string, R extends UploadResultBase>(
+  opts: StoreOptions<M, C, P, R>,
+): StoreRuntime<M, C, P, R>['opts'] {
+  return {
+    ...opts,
+    config: resolveUploadConfig(opts.config),
+    transport: opts.transport ?? createXHRTransport(),
+  }
+}
+
+function emitInternalEvent<M extends IntentMap, C extends CursorMap<M>, P extends string, R extends UploadResultBase>(
+  rt: StoreRuntime<M, C, P, R>,
+  event: InternalEvent<M, C, P, R>,
+) {
+  switch (event.type) {
+    case 'files.added': {
+      for (const item of event.items) {
+        rt.emitter.emit('file.added', {
+          localId: item.localId,
+          purpose: item.purpose,
+          file: item.file,
+          fingerprint: item.fingerprint,
+        })
+      }
+      break
+    }
+    case 'validation.ok': {
+      rt.emitter.emit('validation.ok', { localId: event.localId })
+      rt.emitter.emit('intent.creating', { localId: event.localId })
+      break
+    }
+    case 'validation.failed': {
+      rt.emitter.emit('validation.failed', { localId: event.localId, reason: event.reason })
+      break
+    }
+    case 'intent.ok': {
+      rt.emitter.emit('intent.created', { localId: event.localId, intent: event.intent })
+      break
+    }
+    case 'intent.failed': {
+      rt.emitter.emit('intent.failed', { localId: event.localId, error: event.error, retryable: event.retryable })
+      break
+    }
+    case 'upload.begin': {
+      rt.emitter.emit('upload.started', { localId: event.localId })
+      break
+    }
+    case 'upload.progress': {
+      const pct = event.totalBytes > 0 ? Math.min(100, (event.uploadedBytes / event.totalBytes) * 100) : 0
+      rt.emitter.emit('upload.progress', {
+        localId: event.localId,
+        pct,
+        uploadedBytes: event.uploadedBytes,
+        totalBytes: event.totalBytes,
+      })
+      break
+    }
+    case 'cursor.updated': {
+      rt.emitter.emit('upload.cursor', { localId: event.localId, cursor: event.cursor })
+      break
+    }
+    case 'upload.ok': {
+      rt.emitter.emit('upload.completing', { localId: event.localId })
+      break
+    }
+    case 'upload.failed': {
+      rt.emitter.emit('upload.error', { localId: event.localId, error: event.error, retryable: event.retryable })
+      break
+    }
+    case 'dedupe.ok': {
+      rt.emitter.emit('upload.completed', {
+        localId: event.localId,
+        result: event.result,
+        completedBy: 'dedupe',
+      })
+      break
+    }
+    case 'complete.ok': {
+      rt.emitter.emit('upload.completed', {
+        localId: event.localId,
+        result: event.result,
+        completedBy: 'upload',
+      })
+      break
+    }
+    case 'complete.failed': {
+      rt.emitter.emit('upload.error', { localId: event.localId, error: event.error, retryable: event.retryable })
+      break
+    }
+    case 'paused': {
+      rt.emitter.emit('upload.paused', { localId: event.localId, cursor: event.cursor })
+      break
+    }
+    case 'canceled': {
+      rt.emitter.emit('upload.canceled', { localId: event.localId })
+      break
+    }
+    case 'fingerprint.updated': {
+      break
+    }
+  }
+}
+
+function emitCommandEvents<M extends IntentMap, C extends CursorMap<M>, P extends string, R extends UploadResultBase>(
+  rt: StoreRuntime<M, C, P, R>,
+  cmd: UploadCommand<P>,
+  prev: UploadState<M, C, P, R>,
+  next: UploadState<M, C, P, R>,
+) {
+  if (cmd.type === 'start' || cmd.type === 'resume') {
+    const prevItem = prev.items.get(cmd.localId)
+    const nextItem = next.items.get(cmd.localId)
+    if (!prevItem || !nextItem) return
+    if (nextItem.phase !== 'queued') return
+
+    if (prevItem.phase === 'paused') {
+      rt.emitter.emit('upload.resumed', { localId: cmd.localId })
+    } else {
+      rt.emitter.emit('upload.queued', { localId: cmd.localId })
+    }
+  }
+
+  if (cmd.type === 'cancel') {
+    const prevItem = prev.items.get(cmd.localId)
+    const nextItem = next.items.get(cmd.localId)
+    if (!prevItem || !nextItem) return
+    if (nextItem.phase === 'canceled' && prevItem.phase !== 'canceled') {
+      rt.emitter.emit('upload.canceled', { localId: cmd.localId })
+    }
+  }
+}

package/src/core/engine/store/store.schedule.ts

@@ -0,0 +1,116 @@
+import type { CursorMap, IntentMap, UploadResultBase } from '../../contracts'
+import type { UploadItem } from '../internal-events.types'
+import { createIntent } from './handlers/create-intent'
+import { finalizeUpload } from './handlers/finalize'
+import { runUpload } from './handlers/run-upload'
+import { isAutoStart } from './store.libs'
+import type { StoreRuntime } from './store.types'
+
+/**
+ * Central scheduler that ensures the system is always making progress towards a consistent state.
+ * It checks for:
+ * 1. Pending intent creations
+ * 2. Available upload slots (concurrency limits)
+ * 3. Pending completions
+ *
+ * This function is idempotent and safe to call multiple times.
+ */
+export function scheduleWork<M extends IntentMap, C extends CursorMap<M>, P extends string, R extends UploadResultBase>(
+  rt: StoreRuntime<M, C, P, R>,
+) {
+  if (rt.scheduling) return
+  rt.scheduling = true
+  try {
+    scheduleIntentCreations(rt)
+    scheduleUploads(rt)
+    scheduleCompletes(rt)
+  } finally {
+    rt.scheduling = false
+  }
+}
+
+/**
+ * Enqueues intent creation for items that are in the `creating_intent` phase.
+ *
+ * The intent creation itself runs as an async effect and is guarded by
+ * {@link StoreRuntime.inflightIntents}.
+ */
+export function scheduleIntentCreations<
+  M extends IntentMap,
+  C extends CursorMap<M>,
+  P extends string,
+  R extends UploadResultBase,
+>(rt: StoreRuntime<M, C, P, R>) {
+  for (const item of rt.state.items.values()) {
+    if (item.phase !== 'creating_intent') continue
+    if (rt.inflightIntents.has(item.localId)) continue
+    rt.enqueueEffect(() => createIntent(rt, item.localId))
+  }
+}
+
+/**
+ * Enqueues completion/finalization calls for items in the `completing` phase.
+ *
+ * Finalization runs as an async effect and is guarded by {@link StoreRuntime.inflightCompletes}.
+ */
+export function scheduleCompletes<
+  M extends IntentMap,
+  C extends CursorMap<M>,
+  P extends string,
+  R extends UploadResultBase,
+>(rt: StoreRuntime<M, C, P, R>) {
+  for (const item of rt.state.items.values()) {
+    if (item.phase !== 'completing') continue
+    if (rt.inflightCompletes.has(item.localId)) continue
+    rt.enqueueEffect(() => finalizeUpload(rt, item.localId))
+  }
+}
+
+/**
+ * Manages upload concurrency and starts new uploads when slots are available.
+ * Respects `maxConcurrentUploads` from config.
+ */
+export function scheduleUploads<
+  M extends IntentMap,
+  C extends CursorMap<M>,
+  P extends string,
+  R extends UploadResultBase,
+>(rt: StoreRuntime<M, C, P, R>) {
+  const maxConcurrent = Math.max(1, rt.opts.config.maxConcurrentUploads)
+  const active = rt.inflightUploads.size
+
+  // Auto-start items
+  for (const item of rt.state.items.values()) {
+    if (item.phase === 'ready' && isAutoStart(rt.opts, item.purpose)) {
+      rt.applyCommand({ type: 'start', localId: item.localId })
+    }
+  }
+
+  if (active >= maxConcurrent) return
+
+  // Find queued items
+  const candidates = Array.from(rt.state.items.values()).filter(isQueuedItem)
+  if (candidates.length === 0) return
+
+  const slots = Math.max(0, maxConcurrent - active)
+  const toStart = candidates.slice(0, slots)
+
+  for (const item of toStart) {
+    // Guard against rare scheduling races.
+    if (rt.inflightUploads.has(item.localId)) continue
+
+    const controller = new AbortController()
+    rt.inflightUploads.set(item.localId, { controller, mode: 'normal', started: false })
+
+    rt.applyInternal({ type: 'upload.begin', localId: item.localId, startedAt: Date.now() })
+
+    // Fire-and-forget: runUpload is responsible for updating state and cleaning inflight entries.
+    void runUpload(rt, item.localId)
+  }
+}
+
+function isQueuedItem<M extends IntentMap, C extends CursorMap<M>, P extends string, R extends UploadResultBase>(
+  item: UploadItem<M, C, P, R>,
+): item is Extract<UploadItem<M, C, P, R>, { phase: 'queued' }> {
+  return item.phase === 'queued'
+}