@acms-subir/core 0.3.1 → 0.3.3-next.20260929113209

package/src/components/import/runner.ts

@@ -0,0 +1,284 @@
+// Browser half of the import pipeline: read the source file, build the
+// Import Bundle, re-host its files, then drive /api/import step by step.
+// Progress is journaled in localStorage (keyed by a hash of the file) so a
+// closed tab resumes where it stopped instead of re-uploading everything.
+
+import type { Media } from '../../lib/schema';
+import { ImportBundle, resolveAssets, type ImportAsset, type ResolvedAsset } from '../../lib/import/bundle';
+import { ghostToBundle, isGhostExport } from '../../lib/import/ghost';
+import { openZip, type ZipIndex } from '../../lib/import/zipread';
+import { renderMarkdown } from '../../lib/markdown';
+import { DEFAULT_SIZES, mediaSrc, mediaSrcset, mediaUrl, uploadFile, uploadImage } from '../editor/imaging';
+
+export interface Source {
+  kind: ImportBundle['source']['kind'];
+  label: string;
+  bundle: ImportBundle;
+  // acms exports: the ZIP holding media/<key> binaries
+  zip: ZipIndex | null;
+  hash: string;
+}
+
+export interface ImportOptionsState {
+  conflict: 'skip' | 'update' | 'copy';
+  identity: boolean;
+  nav: boolean;
+  redirects: boolean;
+  headCode: boolean;
+  authors: Record<string, string>;
+}
+
+export interface Progress {
+  phase: 'assets' | 'start' | 'items' | 'finish' | 'done';
+  done: number;
+  total: number;
+  label: string;
+}
+
+async function sha(buf: ArrayBuffer): Promise<string> {
+  const d = await crypto.subtle.digest('SHA-256', buf);
+  return [...new Uint8Array(d)].slice(0, 12).map((b) => b.toString(16).padStart(2, '0')).join('');
+}
+
+export async function readSource(file: File, siteUrl = ''): Promise<Source> {
+  const buf = await file.arrayBuffer();
+  const hash = await sha(buf);
+  const isZip = new Uint8Array(buf.slice(0, 4)).join(',') === '80,75,3,4';
+  if (isZip) {
+    const zip = openZip(buf);
+    if (zip.has('content.json') && !zip.has('bundle.json')) {
+      throw new Error('This is an acms backup. Restore it from Settings → Backup instead; it replaces the whole site.');
+    }
+    if (zip.has('bundle.json')) {
+      const bundle = ImportBundle.parse(JSON.parse(new TextDecoder().decode(await zip.read('bundle.json'))));
+      return { kind: bundle.source.kind, label: bundle.source.title || file.name, bundle, zip, hash };
+    }
+    // a Ghost export zipped together with its content folder
+    const json = zip.names.find((n) => /\.json$/i.test(n) && !n.includes('/'));
+    if (json) {
+      const parsed = JSON.parse(new TextDecoder().decode(await zip.read(json)));
+      if (isGhostExport(parsed)) {
+        const bundle = ghostToBundle(parsed, { siteUrl, markdown: renderMarkdown });
+        return { kind: 'ghost', label: bundle.source.title || file.name, bundle, zip, hash };
+      }
+    }
+    throw new Error('This ZIP isn’t an acms export or a Ghost export. For a Ghost images folder, add it under “Images” after choosing the Ghost .json file.');
+  }
+  let parsed: unknown;
+  try { parsed = JSON.parse(new TextDecoder().decode(buf)); }
+  catch { throw new Error('Choose a Ghost export (.json) or an acms export (.zip).'); }
+  if (isGhostExport(parsed)) {
+    const bundle = ghostToBundle(parsed, { siteUrl, markdown: renderMarkdown });
+    return { kind: 'ghost', label: bundle.source.title || file.name, bundle, zip: null, hash };
+  }
+  const format = (parsed as { format?: string })?.format;
+  if (format === 'acms-backup') throw new Error('This is an acms backup. Restore it from Settings → Backup instead.');
+  if (format === 'acms-export') {
+    const bundle = ImportBundle.parse(parsed);
+    return { kind: bundle.source.kind, label: bundle.source.title || file.name, bundle, zip: null, hash };
+  }
+  throw new Error('This file isn’t a format acms can import yet. Supported: Ghost (.json) and acms exports (.zip).');
+}
+
+// ---- asset lookup ----
+
+const MIME: Record<string, string> = {
+  jpg: 'image/jpeg', jpeg: 'image/jpeg', png: 'image/png', gif: 'image/gif', webp: 'image/webp', avif: 'image/avif', svg: 'image/svg+xml',
+  pdf: 'application/pdf', zip: 'application/zip', txt: 'text/plain', csv: 'text/csv', md: 'text/markdown', json: 'application/json',
+  doc: 'application/msword', docx: 'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
+  xls: 'application/vnd.ms-excel', xlsx: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
+  ppt: 'application/vnd.ms-powerpoint', pptx: 'application/vnd.openxmlformats-officedocument.presentationml.presentation',
+  epub: 'application/epub+zip',
+};
+const mimeFor = (name: string, fallback = 'application/octet-stream') => MIME[name.split('.').pop()?.toLowerCase() ?? ''] ?? fallback;
+
+// A Ghost content ZIP can be rooted anywhere: content/images/…, images/…, or
+// inside a site folder (mysite/content/images/…). Match on the tail.
+export function findInZip(zip: ZipIndex, path: string): string | null {
+  if (zip.has(path)) return path;
+  const tail = path.replace(/^content\//, '');
+  return zip.names.find((n) => n === tail || n.endsWith(`/${path}`) || n.endsWith(`/${tail}`)) ?? null;
+}
+
+export function sourceAssetUrl(bundle: ImportBundle, asset: ImportAsset): string {
+  if (/^https?:\/\//.test(asset.url)) return asset.url;
+  return bundle.source.siteUrl ? `${bundle.source.siteUrl.replace(/\/+$/, '')}/${asset.url}` : `/${asset.url}`;
+}
+
+async function fetchAsset(bundle: ImportBundle, asset: ImportAsset, contentZip: ZipIndex | null): Promise<File> {
+  const name = asset.filename || asset.url.split('/').pop() || 'file';
+  const inZip = contentZip && findInZip(contentZip, asset.url);
+  if (inZip) {
+    const bytes = await contentZip!.read(inZip);
+    return new File([bytes], name, { type: mimeFor(inZip) });
+  }
+  const url = sourceAssetUrl(bundle, asset);
+  if (!/^https?:\/\//.test(url)) throw new Error('Not in the uploaded images ZIP, and no old site URL to download from.');
+  const res = await fetch('/api/import/asset', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ url }) });
+  if (!res.ok) throw new Error(((await res.json().catch(() => null)) as { error?: string } | null)?.error ?? `Download failed (${res.status})`);
+  const blob = await res.blob();
+  return new File([blob], name, { type: mimeFor(name, blob.type || 'application/octet-stream') });
+}
+
+// ---- journal ----
+
+interface Journal {
+  runId: string;
+  assets: Record<string, ResolvedAsset & { mediaId: string; sourceId: string }>;
+  failed: Record<string, string>;
+  done: string[];
+  authorIds: Record<string, string | null>;
+}
+const journalKey = (hash: string) => `acms-import:${hash}`;
+export function readJournal(hash: string): Journal | null {
+  try { return JSON.parse(localStorage.getItem(journalKey(hash)) ?? 'null'); } catch { return null; }
+}
+function writeJournal(hash: string, j: Journal) {
+  try { localStorage.setItem(journalKey(hash), JSON.stringify(j)); } catch { /* quota/private mode: no resume */ }
+}
+export function clearJournal(hash: string) {
+  try { localStorage.removeItem(journalKey(hash)); } catch { /* ignore */ }
+}
+
+async function api<T>(body: unknown): Promise<T> {
+  const res = await fetch('/api/import', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(body) });
+  const data = await res.json().catch(() => null);
+  if (!res.ok || !data) throw new Error((data as { error?: string } | null)?.error ?? `Import request failed (${res.status})`);
+  return data as T;
+}
+
+// The plan needs only item headers: bodies can be megabytes.
+export function lite(bundle: ImportBundle): ImportBundle {
+  return { ...bundle, items: bundle.items.map((i) => ({ ...i, html: '' })) };
+}
+
+export interface RunResult {
+  runId: string;
+  report: Record<string, unknown>;
+}
+
+export async function runImport(
+  source: Source,
+  options: ImportOptionsState,
+  contentZip: ZipIndex | null,
+  onProgress: (p: Progress) => void,
+  signal: { cancelled: boolean },
+): Promise<RunResult> {
+  const { bundle, hash } = source;
+  // a Ghost export zipped together with its content folder serves both roles
+  const files = contentZip ?? (source.kind === 'ghost' ? source.zip : null);
+  const journal: Journal = readJournal(hash) ?? { runId: '', assets: {}, failed: {}, done: [], authorIds: {} };
+  const save = () => writeJournal(hash, journal);
+
+  // 1. files → media library
+  const assets = bundle.assets.filter((a) => a.kind !== 'media');
+  const total = assets.length + bundle.media.length;
+  let n = 0;
+  for (const a of assets) {
+    if (signal.cancelled) throw new Error('Import paused. Run it again to continue where it stopped.');
+    onProgress({ phase: 'assets', done: n++, total, label: a.filename || a.url });
+    if (journal.assets[a.ref]) continue;
+    try {
+      const file = await fetchAsset(bundle, a, files);
+      const media: Media = a.kind === 'image' && file.type.startsWith('image/') ? await uploadImage(file, a.alt) : await uploadFile(file);
+      journal.assets[a.ref] = media.mime.startsWith('image/')
+        ? { url: mediaSrc(media), srcset: mediaSrcset(media), sizes: media.variants.length ? DEFAULT_SIZES : '', mediaId: media.id, sourceId: a.url }
+        : { url: mediaUrl(media.key), mediaId: media.id, sourceId: a.url };
+      delete journal.failed[a.ref];
+    } catch (e) {
+      journal.failed[a.ref] = e instanceof Error ? e.message : 'Download failed';
+    }
+    save();
+  }
+  // acms exports: media rows move verbatim with their keys
+  for (const m of bundle.media) {
+    if (signal.cancelled) throw new Error('Import paused. Run it again to continue where it stopped.');
+    onProgress({ phase: 'assets', done: n++, total, label: m.filename });
+    if (journal.assets[`m-${m.id}`] || !source.zip) continue;
+    try {
+      const form = new FormData();
+      form.set('media', JSON.stringify(m));
+      for (const key of [m.key, ...m.variants.map((v) => v.key)]) {
+        form.set(key, new File([await source.zip.read(`media/${key}`)], key.split('/').pop() ?? 'file', { type: key === m.key ? m.mime : 'image/webp' }));
+      }
+      const res = await fetch('/api/import/media', { method: 'POST', body: form });
+      const data = (await res.json().catch(() => null)) as { error?: string; existed?: boolean } | null;
+      if (!res.ok) throw new Error(data?.error ?? `Upload failed (${res.status})`);
+      journal.assets[`m-${m.id}`] = { url: mediaUrl(m.key), mediaId: data?.existed ? '' : m.id, sourceId: m.id };
+      delete journal.failed[`m-${m.id}`];
+    } catch (e) {
+      journal.failed[`m-${m.id}`] = e instanceof Error ? e.message : 'Upload failed';
+    }
+    save();
+  }
+
+  const resolved = new Map(Object.entries(journal.assets).map(([ref, r]) => [ref, r]));
+  const fallback = new Map(bundle.assets.map((a) => [a.ref, sourceAssetUrl(bundle, a)]));
+  const fix = (v: string) => resolveAssets(v, resolved, fallback);
+
+  // 2. run + authors + tags; only people with a byline become authors
+  const writers = new Set(bundle.items.flatMap((i) => i.authors));
+  onProgress({ phase: 'start', done: 0, total: 1, label: 'Authors and tags' });
+  if (!journal.runId || !(await runIsOpen(journal.runId))) {
+    const started = await api<{ runId: string; authorIds: Record<string, string | null> }>({
+      op: 'start', source: bundle.source.kind, label: source.label,
+      authors: bundle.authors.filter((a) => writers.has(a.sourceId)).map((a) => ({ ...a, avatar: fix(a.avatar) })),
+      tags: bundle.tags,
+      media: Object.values(journal.assets).filter((r) => r.mediaId).map((r) => ({ sourceId: r.sourceId, id: r.mediaId })),
+      options,
+    });
+    journal.runId = started.runId;
+    journal.authorIds = started.authorIds;
+    journal.done = [];
+    save();
+  }
+
+  // 3. posts & pages, in batches that stay well under the request cap
+  const items = bundle.items.filter((i) => !journal.done.includes(i.sourceId));
+  let done = bundle.items.length - items.length;
+  const batches: typeof items[] = [];
+  let cur: typeof items = [], size = 0;
+  for (const it of items) {
+    const s = it.html.length + 2000;
+    if (cur.length && (cur.length >= 10 || size + s > 4_000_000)) { batches.push(cur); cur = []; size = 0; }
+    cur.push(it); size += s;
+  }
+  if (cur.length) batches.push(cur);
+  for (const batch of batches) {
+    if (signal.cancelled) throw new Error('Import paused. Run it again to continue where it stopped.');
+    onProgress({ phase: 'items', done, total: bundle.items.length, label: batch[0].title });
+    await api({
+      op: 'items', runId: journal.runId, authorIds: journal.authorIds, options,
+      items: batch.map((i) => ({ ...i, html: fix(i.html), cover: fix(i.cover) })),
+    });
+    journal.done.push(...batch.map((i) => i.sourceId));
+    done += batch.length;
+    save();
+  }
+
+  // 4. redirects, navigation, identity, report
+  onProgress({ phase: 'finish', done: 0, total: 1, label: 'Redirects and settings' });
+  const identity = Object.fromEntries(Object.entries(bundle.identity).map(([k, v]) => [k, typeof v === 'string' ? fix(v) : v]));
+  const failedAssets = Object.entries(journal.failed).map(([ref, error]) => {
+    const a = bundle.assets.find((x) => x.ref === ref);
+    return { url: a ? sourceAssetUrl(bundle, a) : ref.replace(/^m-/, 'media '), error };
+  });
+  const { report } = await api<{ report: Record<string, unknown> }>({
+    op: 'finish', runId: journal.runId, options, identity,
+    nav: bundle.nav.map((n) => ({ ...n, url: fix(n.url) })),
+    legacy: bundle.items.filter((i) => i.legacyPath).map((i) => ({ sourceId: i.sourceId, type: i.type, path: i.legacyPath })),
+    redirects: bundle.redirects,
+    warnings: bundle.warnings,
+    failedAssets,
+  });
+  clearJournal(hash);
+  onProgress({ phase: 'done', done: 1, total: 1, label: '' });
+  return { runId: journal.runId, report };
+}
+
+async function runIsOpen(runId: string): Promise<boolean> {
+  const res = await fetch('/api/import');
+  if (!res.ok) return false;
+  const { runs } = (await res.json()) as { runs: { id: string; status: string }[] };
+  return runs.some((r) => r.id === runId && r.status === 'running');
+}

package/src/lib/embeds.ts

@@ -0,0 +1,50 @@
+// Embeddable-URL detection shared by the editor's embed card (paste →
+// card) and the importers (a Ghost/WordPress embed → embedCard markup).
+export interface EmbedInfo {
+  provider: string;
+  embedSrc: string;
+  aspect: boolean; // true → 16:9, false → fixed height
+}
+
+export function detectEmbed(raw: string): EmbedInfo | null {
+  let url: URL;
+  try {
+    url = new URL(raw.trim());
+  } catch {
+    return null;
+  }
+  const host = url.hostname.replace(/^www\./, '');
+  if (host === 'youtube.com' || host === 'm.youtube.com' || host === 'youtu.be' || host === 'youtube-nocookie.com') {
+    const id =
+      host === 'youtu.be'
+        ? url.pathname.slice(1).split('/')[0]
+        : url.pathname.startsWith('/shorts/') || url.pathname.startsWith('/embed/') || url.pathname.startsWith('/live/')
+          ? url.pathname.split('/')[2]
+          : url.searchParams.get('v');
+    // keep a start offset (?start=90, ?t=1h2m3s) — sermon and talk links
+    // usually point into the middle of a stream
+    const start = youtubeStart(url.searchParams.get('start') ?? url.searchParams.get('t'));
+    if (id) return { provider: 'youtube', embedSrc: `https://www.youtube-nocookie.com/embed/${id}${start ? `?start=${start}` : ''}`, aspect: true };
+  }
+  if (host === 'vimeo.com' || host === 'player.vimeo.com') {
+    const id = url.pathname.split('/').filter(Boolean).pop();
+    if (id && /^\d+$/.test(id)) return { provider: 'vimeo', embedSrc: `https://player.vimeo.com/video/${id}`, aspect: true };
+  }
+  if (host === 'loom.com' && url.pathname.startsWith('/share/')) {
+    return { provider: 'loom', embedSrc: `https://www.loom.com/embed/${url.pathname.split('/')[2]}`, aspect: true };
+  }
+  if (host === 'codepen.io' && url.pathname.includes('/pen/')) {
+    return { provider: 'codepen', embedSrc: raw.replace('/pen/', '/embed/'), aspect: false };
+  }
+  if (url.protocol === 'https:') {
+    return { provider: 'iframe', embedSrc: raw, aspect: false };
+  }
+  return null;
+}
+
+function youtubeStart(raw: string | null): number {
+  if (!raw) return 0;
+  if (/^\d+s?$/.test(raw)) return parseInt(raw, 10);
+  const m = raw.match(/^(?:(\d+)h)?(?:(\d+)m)?(?:(\d+)s)?$/);
+  return m ? Number(m[1] ?? 0) * 3600 + Number(m[2] ?? 0) * 60 + Number(m[3] ?? 0) : 0;
+}