@michaelthielemann/kestrel 3.0.0 → 4.0.0

package/layers/media/server/utils/storage-relocate.ts

@@ -1,139 +0,0 @@
-import { eq, getTableColumns } from 'drizzle-orm'
-import { createError } from 'h3'
-import type { BetterSQLite3Database } from 'drizzle-orm/better-sqlite3'
-import { buildKey } from './naming'
-import { derivativeKey, type DerivativeManifest } from './record'
-import { media } from '../collections/media'
-import { emitMediaWrite } from './media-write'
-import type { StorageDriver } from '../../../core/server/utils/storage'
-import { withLock, mediaLockKey } from '../../../core/server/utils/key-lock'
-
-export interface RelocatePlan {
-  storageKey: string
-  derivatives: DerivativeManifest
-  copies: { src: string; dst: string }[]
-}
-
-/** Pure: compute the new storageKey, the rewritten derivative manifest, and the list of object
- *  copies (original + each derivative) for moving/renaming a media row to `folder`/`filename`. */
-export function planObjectRelocation(
-  row: { storageKey: string; derivatives: DerivativeManifest | null },
-  folder: string,
-  filename: string,
-): RelocatePlan {
-  const storageKey = buildKey(folder, filename)
-  const derivatives: DerivativeManifest = {}
-  const copies: { src: string; dst: string }[] = [{ src: row.storageKey, dst: storageKey }]
-  for (const [manifestKey, entry] of Object.entries(row.derivatives ?? {})) {
-    // manifestKey is `<name>.<format>`; variant names are `[A-Za-z0-9_-]` (no dots), so the LAST dot
-    // splits name from format cleanly — rebuild the destination key from the name, not the (now-absent
-    // from the key scheme) width.
-    const dot = manifestKey.lastIndexOf('.')
-    const name = dot > 0 ? manifestKey.slice(0, dot) : manifestKey
-    const format = dot > 0 ? manifestKey.slice(dot + 1) : (entry.mime.split('/').pop() ?? 'bin')
-    const dst = derivativeKey(storageKey, name, format)
-    derivatives[manifestKey] = { ...entry, key: dst }
-    copies.push({ src: entry.key, dst })
-  }
-  return { storageKey, derivatives, copies }
-}
-
-interface MediaRowLite { id: number; storageKey: string; folder: string | null; filename: string; derivatives: DerivativeManifest | null }
-
-function mediaCols() { return getTableColumns(media) as Record<string, never> }
-
-/** Reject if any destination object already exists on disk (e.g. another media's derivative, which
- *  carries no `storageKey` row, so the DB clash check misses it). Keys the plan itself vacates are
- *  excluded so a plain move never collides with its own source. */
-async function assertDestKeysFree(driver: StorageDriver, plan: RelocatePlan): Promise<void> {
-  const sources = new Set(plan.copies.map((c) => c.src))
-  for (const c of plan.copies) {
-    if (sources.has(c.dst)) continue
-    if (await driver.exists?.(c.dst)) {
-      throw createError({ statusCode: 409, statusMessage: `A file already exists at ${c.dst}` })
-    }
-  }
-}
-
-/** Move and/or rename a media object (+ its derivatives). Pre-checks the destination key (409 if
- *  taken), copies to the new keys, updates the row, then deletes the old keys; on a DB failure the
- *  freshly-copied objects are removed so the source is never lost. No-op if the target equals the
- *  current key. */
-export async function relocateMedia(
-  db: BetterSQLite3Database, driver: StorageDriver, id: number,
-  target: { folder?: string; filename?: string },
-): Promise<void> {
-  const cols = mediaCols()
-  const row = db.select().from(media).where(eq(cols.id, id)).get() as MediaRowLite | undefined
-  if (!row) throw createError({ statusCode: 404, statusMessage: `media ${id} not found` })
-
-  const folder = target.folder ?? row.folder ?? ''
-  const filename = target.filename ?? row.filename
-  const plan = planObjectRelocation(row, folder, filename)
-  if (plan.storageKey === row.storageKey) return
-
-  // Same lock the upload path holds across its own exists-check → put → insert (mediaLockKey, keyed on
-  // the DESTINATION key only): without it, a concurrent upload or another relocate targeting this same
-  // key can observe it free mid-way through this one and land on top of it — or, two relocates racing
-  // for the same destination, have the loser's failure-compensation delete the WINNER's just-landed blob.
-  await withLock(mediaLockKey(plan.storageKey), async () => {
-    const clash = db.select({ id: cols.id }).from(media).where(eq(cols.storageKey, plan.storageKey)).get() as { id: number } | undefined
-    if (clash) throw createError({ statusCode: 409, statusMessage: `A file already exists at ${plan.storageKey}` })
-    await assertDestKeysFree(driver, plan)
-
-    for (const c of plan.copies) await driver.copy(c.src, c.dst)
-    try {
-      db.update(media).set({ storageKey: plan.storageKey, folder: folder || null, filename, derivatives: plan.derivatives, updatedAt: new Date() }).where(eq(cols.id, id)).run()
-    } catch (error) {
-      for (const c of plan.copies) await driver.delete(c.dst)
-      throw error
-    }
-    for (const c of plan.copies) await driver.delete(c.src)
-  })
-  // The public URL is built from storageKey — moving/renaming changes it, so embedding pages must re-render.
-  emitMediaWrite({ id }, { id })
-}
-
-interface MediaRowFull extends MediaRowLite {
-  mime: string; ext: string; size: number; width: number | null; height: number | null
-  checksum: string | null; thumbhash: string | null
-  translations: Record<string, unknown> | null
-}
-
-/** Duplicate a media object (+ derivatives) to a new key and insert a new row (same bytes →
- *  same checksum/dims/translations). Rejects 409 if the target key is taken. */
-export async function duplicateMedia(
-  db: BetterSQLite3Database, driver: StorageDriver, id: number,
-  target: { folder?: string; filename?: string },
-): Promise<{ id: number }> {
-  const cols = mediaCols()
-  const row = db.select().from(media).where(eq(cols.id, id)).get() as MediaRowFull | undefined
-  if (!row) throw createError({ statusCode: 404, statusMessage: `media ${id} not found` })
-
-  const folder = target.folder ?? row.folder ?? ''
-  const filename = target.filename ?? row.filename
-  const plan = planObjectRelocation(row, folder, filename)
-
-  // See relocateMedia: same destination-key lock the upload path holds, so a concurrent upload/relocate
-  // targeting this exact key can't observe it free mid-way through this copy+insert.
-  return withLock(mediaLockKey(plan.storageKey), async () => {
-    const clash = db.select({ id: cols.id }).from(media).where(eq(cols.storageKey, plan.storageKey)).get() as { id: number } | undefined
-    if (clash) throw createError({ statusCode: 409, statusMessage: `A file already exists at ${plan.storageKey}` })
-    await assertDestKeysFree(driver, plan)
-
-    for (const c of plan.copies) await driver.copy(c.src, c.dst)
-    try {
-      const created = db.insert(media).values({
-        storageKey: plan.storageKey, folder: folder || null, filename,
-        mime: row.mime, ext: row.ext, size: row.size, width: row.width, height: row.height,
-        checksum: row.checksum, thumbhash: row.thumbhash, derivatives: plan.derivatives,
-        translations: row.translations ?? {},
-      } as never).returning({ id: cols.id }).get() as { id: number }
-      emitMediaWrite(null, { id: created.id }) // new media row → re-render listings
-      return created
-    } catch (error) {
-      for (const c of plan.copies) await driver.delete(c.dst)
-      throw error
-    }
-  })
-}

package/layers/media/server/utils/translations.ts

@@ -1,19 +0,0 @@
-export type LocaleFields = Record<string, unknown>
-export type Translations = Record<string, LocaleFields>
-
-const isPlainObject = (v: unknown): v is Record<string, unknown> =>
-  typeof v === 'object' && v !== null && !Array.isArray(v)
-
-/** Deep-merge a partial translations patch into the current map: new locales are added, and within a
- *  locale the patched fields are merged over the existing ones — so patching `en.alt` keeps `en.title`
- *  / `en.description`, and other locales are untouched. Defensive against malformed input: a non-object
- *  current/patch or a non-object per-locale value is ignored rather than persisting garbage. */
-export function mergeTranslations(current: Translations | null | undefined, patch: Translations): Translations {
-  const out: Translations = isPlainObject(current) ? { ...current } : {}
-  if (!isPlainObject(patch)) return out
-  for (const [loc, fields] of Object.entries(patch)) {
-    if (!isPlainObject(fields)) continue
-    out[loc] = { ...(isPlainObject(out[loc]) ? out[loc] : {}), ...fields }
-  }
-  return out
-}

package/layers/media/server/utils/usages.ts

@@ -1,66 +0,0 @@
-import { sql, getTableColumns, getTableName, type SQL } from 'drizzle-orm'
-import type { AnySQLiteColumn } from 'drizzle-orm/sqlite-core'
-import type { BetterSQLite3Database } from 'drizzle-orm/better-sqlite3'
-import { allCollections } from '../../../core/server/utils/registry'
-import { fieldIs } from '../../../core/server/utils/defineCollection'
-import { resolveColumnName } from '../../../fields/server/field-registry/naming'
-
-export interface MediaUsage { collection: string; recordId: number; field: string }
-
-/**
- * Reverse lookup for a whole set of media ids at once: ONE query per (collection, column), never one per
- * id. The JSON columns can only be searched by a full table scan, and better-sqlite3 is synchronous, so a
- * per-id loop over a folder-sized selection would block the event loop for seconds. Every requested id
- * gets an entry (empty when unreferenced). Mirrors `findReferrersForMany` in core's record-ref index.
- */
-export function findMediaUsagesForMany(db: BetterSQLite3Database, ids: number[]): Record<number, MediaUsage[]> {
-  const out: Record<number, MediaUsage[]> = {}
-  for (const id of ids) out[id] = []
-  if (!ids.length) return out
-  // The id set travels as ONE bound parameter (a JSON array), not one placeholder per id: a recursive
-  // folder delete can hand us tens of thousands of ids, well past SQLite's 32766-variable ceiling.
-  const idSet = sql`(select value from json_each(${JSON.stringify(ids)}))`
-
-  for (const c of allCollections()) {
-    if (c.name === 'media') continue
-    const cols = getTableColumns(c.table) as Record<string, AnySQLiteColumn>
-    const tbl = sql.identifier(getTableName(c.table))
-    const rowId = sql.identifier(cols.id.name)
-    const collect = (query: SQL, field: string) => {
-      for (const r of db.all(query) as { rid: number; mid: number }[]) {
-        out[r.mid]?.push({ collection: c.name, recordId: r.rid, field })
-      }
-    }
-    // Media ids can nest anywhere inside a repeater/json/content/seo value — json_tree the whole column and
-    // match any integer atom (a safe over-approximation for usage warnings).
-    const scanJson = (column: string, field: string) => collect(
-      sql`select distinct t.${rowId} as rid, jt.atom as mid from ${tbl} t, json_tree(t.${sql.identifier(column)}) jt
-          where jt.type = 'integer' and jt.atom in ${idSet}`,
-      field,
-    )
-
-    for (const [key, field] of Object.entries(c.def.fields)) {
-      if (fieldIs(field, 'media') && !field.options?.multiple) {
-        const { jsKey } = resolveColumnName(key, field)
-        const col = sql.identifier(cols[jsKey].name)
-        collect(sql`select t.${rowId} as rid, t.${col} as mid from ${tbl} t where t.${col} in ${idSet}`, key)
-      } else if (fieldIs(field, 'media') && field.options?.multiple) {
-        const col = sql.identifier(cols[key].name)
-        collect(sql`select distinct t.${rowId} as rid, je.value as mid from ${tbl} t, json_each(t.${col}) je
-                    where je.value in ${idSet}`, key)
-      } else if (field.type === 'repeater' || field.type === 'json') {
-        const { jsKey } = resolveColumnName(key, field)
-        scanJson(cols[jsKey].name, key)
-      }
-    }
-    if (c.def.blocks?.enabled) scanJson('content', 'content')
-    // The `seo` system column carries the social-share image, so a page whose only reference to an asset is
-    // its og:image must still be reported — otherwise deleting it silently 404s the published meta tag.
-    if (c.def.seo) scanJson('seo', 'seo.image')
-  }
-  return out
-}
-
-export function findMediaUsages(db: BetterSQLite3Database, id: number): MediaUsage[] {
-  return findMediaUsagesForMany(db, [id])[id]!
-}

package/layers/media/server/utils/variant-capture.ts

@@ -1,70 +0,0 @@
-import { eq, getTableColumns } from 'drizzle-orm'
-import type { BetterSQLite3Database } from 'drizzle-orm/better-sqlite3'
-import type { ResolvedVariant } from '../../../core/server/utils/kestrel-config'
-import { mediaSettings } from '../collections/media-settings'
-import { reconcileVariants, type StoredVariant } from './variants'
-
-// Auto-discovery accumulator. `KestrelImg`/`useMediaVariant` stash their concrete specs on the render's
-// `event.context.kestrelVariants`; a `beforeResponse` hook (only for publish renders, gated on the
-// `x-kestrel-publish` header) drains them into this module Map, keyed by name and unioning formats across
-// usages. At the end of a FULL publish / generate `saveDiscoveredVariants` reconciles the union into the
-// registry and clears it — so the registry reflects exactly what the last full render actually used.
-const discovered = new Map<string, ResolvedVariant>()
-
-/** Merge freshly-declared specs into the accumulator (union formats for same-named variants). */
-export function recordVariants(specs: readonly ResolvedVariant[]): void {
-  for (const s of specs) {
-    if (!s?.name) continue
-    const prev = discovered.get(s.name)
-    if (prev) prev.formats = [...new Set([...prev.formats, ...(s.formats ?? [])])]
-    else discovered.set(s.name, { ...s, formats: [...(s.formats ?? ['webp'])] })
-  }
-}
-
-export function collectVariants(): ResolvedVariant[] {
-  return [...discovered.values()]
-}
-
-export function clearVariants(): void {
-  discovered.clear()
-}
-
-/**
- * Reconcile the accumulated discovered specs into the `media_settings` registry (scan entries replaced,
- * manual/pinned kept) and clear the accumulator. Called at the end of a full publish / generate.
- *
- * Safety: if NOTHING was discovered this run, leave the registry untouched rather than wiping every scan
- * entry — a zero result almost always means the capture didn't fire (no KestrelImg rendered, or a plumbing
- * gap), not that the site genuinely uses no variants. Narrowing still works whenever ≥1 variant is found.
- *
- * `{ clear: false }` reconciles WITHOUT draining the accumulator — the classic `nuxt generate` topology has
- * no publisher to run this once at the end, so it reconciles after each prerendered route and the growing
- * accumulator converges to the full used set by the last route.
- */
-export function saveDiscoveredVariants(db: BetterSQLite3Database, opts: { clear?: boolean } = {}): void {
-  const found = collectVariants()
-  if (!found.length) { if (opts.clear !== false) clearVariants(); return }
-  const cols = getTableColumns(mediaSettings) as Record<string, never>
-  try {
-    // The registry is a read-modify-write, and a `nuxt generate` can run against the SAME DB as a live
-    // server's publisher — an unguarded select→reconcile→update loses one side's variants. An IMMEDIATE
-    // transaction takes SQLite's write lock up front, so a concurrent writer WAITS (busy_timeout) and then
-    // reconciles against the already-updated row instead of clobbering it (cross-process safe).
-    db.transaction((tx) => {
-      const row = tx.select().from(mediaSettings).where(eq(cols.singletonKey, 'media_settings')).get() as
-        | { variants?: StoredVariant[] | null }
-        | undefined
-      // The json field type's column default is the OBJECT shape '{}' (not '[]') — a generic singleton
-      // PUT that omits `variants` leaves exactly that, so nullish-only guarding isn't enough here.
-      const next = reconcileVariants(Array.isArray(row?.variants) ? row.variants : [], found)
-      if (row) tx.update(mediaSettings).set({ variants: next, updatedAt: new Date() } as never).where(eq(cols.singletonKey, 'media_settings')).run()
-      else tx.insert(mediaSettings).values({ singletonKey: 'media_settings', variants: next } as never).run()
-    }, { behavior: 'immediate' })
-    // Drain the accumulator only AFTER a durable write, so a transient failure keeps it for the next pass.
-    if (opts.clear !== false) clearVariants()
-  } catch (error) {
-    // media_settings not migrated yet — skip persisting the registry (narrowing stays off, config fallback
-    // applies) rather than crash the publish. See the schema-drift note; run `drizzle-kit generate`.
-    console.warn('[kestrel] media_settings registry not available — variant discovery not persisted:', (error as Error)?.message ?? error)
-  }
-}

package/layers/media/server/utils/variants.ts

@@ -1,123 +0,0 @@
-import { eq, getTableColumns } from 'drizzle-orm'
-import type { BetterSQLite3Database } from 'drizzle-orm/better-sqlite3'
-import type { ResolvedVariant, VariantFit } from '../../../core/server/utils/kestrel-config'
-import { mediaSettings } from '../collections/media-settings'
-
-const VARIANT_FITS = new Set<VariantFit>(['cover', 'contain', 'inside', 'outside', 'fill'])
-// sharp's crop gravity/position values (resize({fit:'cover'|'contain', position})).
-const SHARP_POSITIONS = new Set([
-  'top', 'right top', 'right', 'right bottom', 'bottom', 'left bottom', 'left', 'left top', 'centre', 'center',
-  'north', 'northeast', 'east', 'southeast', 'south', 'southwest', 'west', 'northwest', 'entropy', 'attention',
-])
-
-/**
- * A variant persisted in the `media_settings` registry: a resolved variant plus its provenance. `source`
- * distinguishes an auto-discovered (`'scan'`) entry from a hand-authored (`'manual'`) one; a `manual`/`pinned`
- * entry survives the scan reconcile and wins a name collision with a scanned entry.
- */
-export interface StoredVariant extends ResolvedVariant {
-  source?: 'manual' | 'scan'
-  pinned?: boolean
-}
-
-const rank = (v: StoredVariant): number => (v.pinned || v.source === 'manual' ? 1 : 0)
-
-/**
- * The effective variant set to derive: the stored registry entries (deduped by name, a manual/pinned entry
- * winning a scan collision, stripped of provenance and re-defaulted defensively), or `fallback` (the resolved
- * config policy variants) when the registry is empty/absent. `presets` are config-authored named variants
- * (`image.variants`): explicit, name-referenced declarations that are never scan-discovered, so they stay
- * active through usage-driven narrowing (unioned into a non-empty registry, winning any name collision).
- *
- * `fromRegistry` is the verdict a DELETING caller (the backfill prune) must gate on: true only when at least
- * one stored entry SURVIVED validation, so the set really is the registered one. Absent, unread, empty and
- * wholly-rejected registries all resolve to the same superset fallback, under which every registered
- * derivative looks deregistered — the row count alone cannot tell those apart from a narrowed set.
- * Pure — the load-bearing logic, unit-tested without a DB.
- */
-export function resolveActiveVariants(
-  stored: StoredVariant[] | null | undefined, fallback: ResolvedVariant[], presets: ResolvedVariant[] = [],
-): { specs: ResolvedVariant[]; fromRegistry: boolean } {
-  // A registry row is hand-authorable via the media_settings JSON PATCH; reject a name outside the
-  // derivative-key charset ([A-Za-z0-9_-]) — an out-of-charset char breaks the URL / the prune-media
-  // referencedKeys regex (its derivative is pruned though pages reference it) and a `/` nests a pseudo-folder.
-  const safeName = (n: unknown): n is string => typeof n === 'string' && /^[A-Za-z0-9_-]+$/.test(n)
-  const list = (Array.isArray(stored) ? stored : []).filter(
-    (v): v is StoredVariant => !!v && safeName(v.name) && Number.isFinite(v.width) && v.width >= 1,
-  )
-  // Empty registry ⇒ the fallback already contains the presets (resolveVariants unions them), so return it as-is.
-  if (!list.length) return { specs: fallback, fromRegistry: false }
-  const byName = new Map<string, StoredVariant>()
-  for (const v of list) {
-    const prev = byName.get(v.name)
-    if (!prev || rank(v) > rank(prev)) byName.set(v.name, v)
-  }
-  for (const p of presets) {
-    if (p && typeof p.name === 'string' && p.name !== '' && Number.isFinite(p.width) && p.width >= 1) byName.set(p.name, { ...p, source: 'manual' })
-  }
-  const specs = [...byName.values()].map((v): ResolvedVariant => ({
-    name: v.name,
-    width: Math.floor(v.width),
-    // coerce a garbage/stale height to null so it never reaches sharp's crop resize and 500s the upload
-    height: typeof v.height === 'number' && Number.isFinite(v.height) && v.height >= 1 ? Math.floor(v.height) : null,
-    // A hand-authored (media_settings PATCH) fit/position outside sharp's enum would otherwise reach
-    // sharp.resize() verbatim and throw for EVERY upload that derives this variant — allow-list, don't
-    // just default the nullish case.
-    fit: v.fit && VARIANT_FITS.has(v.fit) ? v.fit : 'cover',
-    position: typeof v.position === 'string' && SHARP_POSITIONS.has(v.position) ? v.position : 'centre',
-    formats: v.formats?.length ? v.formats : ['webp'],
-  }))
-  return { specs, fromRegistry: true }
-}
-
-/**
- * Reconcile a fresh scan into the stored registry: every hand-authored entry (`source:'manual'` or
- * `pinned`) is KEPT, and the discovered specs replace ALL prior `source:'scan'` entries (so a variant whose
- * last `KestrelImg` usage was removed disappears — usage-driven narrowing). A manual/pinned entry wins a
- * name collision with a discovered one. Pure — the load-bearing reconcile, unit-tested without a DB.
- */
-export function reconcileVariants(existing: StoredVariant[], discovered: ResolvedVariant[]): StoredVariant[] {
-  const out: StoredVariant[] = []
-  const seen = new Set<string>()
-  // Belt-and-braces against the json field type's object-shaped default ('{}', not '[]') reaching here
-  // directly — mirrors the Array.isArray guard in resolveActiveVariants.
-  for (const e of (Array.isArray(existing) ? existing : [])) {
-    if ((e.source === 'manual' || e.pinned) && e.name && !seen.has(e.name)) { out.push(e); seen.add(e.name) }
-  }
-  for (const d of discovered) {
-    if (!d.name || seen.has(d.name)) continue
-    out.push({ ...d, source: 'scan' })
-    seen.add(d.name)
-  }
-  return out
-}
-
-/**
- * The raw stored registry rows (the `media_settings` singleton), or `null` when there are none to resolve
- * from — the read threw (media_settings not migrated) or nothing is stored. Whether the resolved set may be
- * DELETED against is not decidable here: that is `resolveActiveVariants`' `fromRegistry` verdict, which only
- * a validation pass over these rows can give.
- */
-export function readVariantRegistry(db: BetterSQLite3Database): StoredVariant[] | null {
-  const cols = getTableColumns(mediaSettings) as Record<string, never>
-  let row: { variants?: StoredVariant[] | null } | undefined
-  try {
-    row = db.select().from(mediaSettings).where(eq(cols.singletonKey, 'media_settings')).get() as
-      | { variants?: StoredVariant[] | null }
-      | undefined
-  } catch {
-    return null
-  }
-  const stored = row?.variants
-  return Array.isArray(stored) ? stored : null
-}
-
-/**
- * Resolves the active variant set for the upload path (narrow generation: registered set + presets).
- * Deliberately forgiving: an unreadable registry degrades to the config fallback rather than 500 the
- * upload (mirrors attachDeadRefs tolerating a missing record_refs). Discards `fromRegistry` — never use
- * this to decide what may be pruned; a deleting caller needs `resolveActiveVariants` directly.
- */
-export function activeVariants(db: BetterSQLite3Database, fallback: ResolvedVariant[], presets: ResolvedVariant[] = []): ResolvedVariant[] {
-  return resolveActiveVariants(readVariantRegistry(db), fallback, presets).specs
-}