@cyber-dash-tech/revela 0.3.0 → 0.4.0

package/lib/media/batch-save.ts

@@ -0,0 +1,146 @@
+import { saveMediaAsset } from "./save"
+import type { MediaPurpose, MediaSaveResult } from "./types"
+
+export interface MediaBatchItem {
+  candidateId: string
+  description: string
+  url: string
+  alt?: string
+  use: "logo" | "portrait" | "screenshot"
+  sourceFile?: string
+  intendedSection?: string
+}
+
+export interface MediaBatchSaveInput {
+  topic: string
+  items: MediaBatchItem[]
+  intendedSection?: string
+}
+
+export interface MediaBatchSaveResultItem {
+  candidateId: string
+  assetId: string
+  status: string
+  path: string | null
+  error?: string
+}
+
+export interface MediaBatchSaveResult {
+  ok: true
+  topic: string
+  manifestPath: string | null
+  saved: number
+  failed: number
+  results: MediaBatchSaveResultItem[]
+}
+
+function candidateKey(candidateId: string): string {
+  return slugify(candidateId) || "item"
+}
+
+function slugify(value: string): string {
+  return value
+    .toLowerCase()
+    .replace(/[^a-z0-9]+/g, "-")
+    .replace(/^-+|-+$/g, "")
+}
+
+function purposeForUse(use: MediaBatchItem["use"]): MediaPurpose {
+  return use
+}
+
+function baseAssetId(item: MediaBatchItem): string {
+  const description = slugify(item.description)
+  const fallback = slugify(item.sourceFile ?? item.candidateId)
+  return description || fallback || "image"
+}
+
+function buildDeterministicAssetIds(items: MediaBatchItem[]): Map<string, string> {
+  const assetIds = new Map<string, string>()
+  for (const item of items) {
+    const base = baseAssetId(item)
+    assetIds.set(item.candidateId, `${base}-${candidateKey(item.candidateId)}`)
+  }
+  return assetIds
+}
+
+export async function batchSaveMediaAssets(
+  input: MediaBatchSaveInput,
+  workspaceDir: string,
+): Promise<MediaBatchSaveResult> {
+  const topic = slugify(input.topic)
+  const results: MediaBatchSaveResultItem[] = []
+  const urlToResult = new Map<string, MediaBatchSaveResultItem>()
+  const assetIds = buildDeterministicAssetIds(input.items)
+  let manifestPath: string | null = null
+  let saved = 0
+  let failed = 0
+
+  for (const item of input.items) {
+    const normalizedUrl = item.url.trim()
+    const existing = urlToResult.get(normalizedUrl)
+    if (existing) {
+      results.push({
+        candidateId: item.candidateId,
+        assetId: existing.assetId,
+        status: existing.status,
+        path: existing.path,
+        error: existing.error,
+      })
+      continue
+    }
+
+    const assetId = assetIds.get(item.candidateId) ?? `${baseAssetId(item)}-01`
+    const brief = item.sourceFile
+      ? `${item.description} from ${item.sourceFile}`
+      : item.description
+    const saveResult: MediaSaveResult = await saveMediaAsset({
+      topic,
+      id: assetId,
+      type: "image",
+      purpose: purposeForUse(item.use),
+      brief,
+      status: "success",
+      intendedSection: item.intendedSection ?? input.intendedSection,
+      sourceUrl: normalizedUrl,
+      alt: item.alt,
+    }, workspaceDir)
+
+    const mapped: MediaBatchSaveResultItem = saveResult.ok
+      ? {
+        candidateId: item.candidateId,
+        assetId,
+        status: saveResult.status,
+        path: saveResult.path,
+      }
+      : {
+        candidateId: item.candidateId,
+        assetId,
+        status: "error",
+        path: null,
+        error: saveResult.error,
+      }
+
+    if (saveResult.ok) manifestPath = saveResult.manifestPath
+    if (mapped.path) saved += 1
+    else failed += 1
+
+    urlToResult.set(normalizedUrl, {
+      candidateId: item.candidateId,
+      assetId,
+      status: mapped.status,
+      path: mapped.path,
+      error: mapped.error,
+    })
+    results.push(mapped)
+  }
+
+  return {
+    ok: true,
+    topic,
+    manifestPath,
+    saved,
+    failed,
+    results,
+  }
+}

package/lib/media/download.ts

@@ -0,0 +1,68 @@
+import { extname } from "path"
+
+const MIME_TO_EXT: Record<string, string> = {
+  "image/png": ".png",
+  "image/jpeg": ".jpg",
+  "image/webp": ".webp",
+  "image/gif": ".gif",
+}
+
+const ALLOWED_EXTENSIONS = new Set([".png", ".jpg", ".jpeg", ".webp", ".gif"])
+
+function normalizeExtension(ext: string): string {
+  const value = ext.toLowerCase()
+  if (value === ".jpeg") return ".jpg"
+  return value
+}
+
+export function inferImageExtension(contentType: string | null, sourceName = ""): string | null {
+  const mimeBase = contentType?.split(";")[0].trim().toLowerCase() ?? ""
+  if (mimeBase) {
+    const byMime = MIME_TO_EXT[mimeBase]
+    if (byMime) return byMime
+    if (!mimeBase.startsWith("image/")) return null
+  }
+
+  const ext = normalizeExtension(extname(sourceName))
+  if (ALLOWED_EXTENSIONS.has(ext)) return ext
+  return null
+}
+
+export async function downloadImageFromUrl(
+  url: string,
+): Promise<{ buffer: Buffer; contentType: string | null; extension: string }> {
+  let parsed: URL
+  try {
+    parsed = new URL(url)
+  } catch {
+    throw new Error("INVALID_URL")
+  }
+
+  if (parsed.protocol !== "http:" && parsed.protocol !== "https:") {
+    throw new Error("INVALID_URL")
+  }
+
+  const response = await fetch(parsed, {
+    headers: {
+      "User-Agent":
+        "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 " +
+        "(KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36",
+    },
+  })
+
+  if (!response.ok) {
+    throw new Error(`DOWNLOAD_FAILED:${response.status}`)
+  }
+
+  const contentType = response.headers.get("content-type")
+  const extension = inferImageExtension(contentType, parsed.pathname)
+  if (!extension) {
+    throw new Error("UNSUPPORTED_IMAGE_TYPE")
+  }
+
+  return {
+    buffer: Buffer.from(await response.arrayBuffer()),
+    contentType,
+    extension,
+  }
+}

package/lib/media/save.ts

@@ -0,0 +1,273 @@
+import {
+  copyFileSync,
+  existsSync,
+  mkdirSync,
+  readFileSync,
+  rmSync,
+  writeFileSync,
+} from "fs"
+import { dirname, join, relative, resolve, sep } from "path"
+import { downloadImageFromUrl, inferImageExtension } from "./download"
+import type {
+  MediaAssetRecord,
+  MediaManifest,
+  MediaSaveInput,
+  MediaSaveResult,
+  MediaStatus,
+} from "./types"
+
+function nowIso(): string {
+  return new Date().toISOString()
+}
+
+function slugify(value: string): string {
+  return value
+    .toLowerCase()
+    .replace(/[^a-z0-9]+/g, "-")
+    .replace(/^-+|-+$/g, "")
+}
+
+function ensureInsideWorkspace(pathValue: string, workspaceDir: string): string {
+  const abs = resolve(workspaceDir, pathValue)
+  const root = resolve(workspaceDir)
+  if (abs !== root && !abs.startsWith(root + sep)) {
+    throw new Error("sourcePath must be within workspace")
+  }
+  return abs
+}
+
+function readManifest(manifestPath: string, topic: string): MediaManifest {
+  if (!existsSync(manifestPath)) {
+    return { topic, updatedAt: nowIso(), assets: [] }
+  }
+
+  const parsed = JSON.parse(readFileSync(manifestPath, "utf-8")) as Partial<MediaManifest>
+  return {
+    topic,
+    updatedAt: typeof parsed.updatedAt === "string" ? parsed.updatedAt : nowIso(),
+    assets: Array.isArray(parsed.assets) ? parsed.assets as MediaAssetRecord[] : [],
+  }
+}
+
+function buildFailureRecord(
+  input: MediaSaveInput,
+  topic: string,
+  status: Exclude<MediaStatus, "success">,
+  failureReason: string,
+  sourcePath?: string,
+): MediaAssetRecord {
+  return {
+    id: slugify(input.id),
+    type: "image",
+    purpose: input.purpose,
+    brief: input.brief,
+    status,
+    path: null,
+    sourceUrl: input.sourceUrl,
+    sourcePath,
+    intendedSection: input.intendedSection,
+    alt: input.alt,
+    notes: input.notes,
+    failureReason,
+    savedAt: nowIso(),
+  }
+}
+
+function upsertAsset(manifest: MediaManifest, record: MediaAssetRecord): { manifest: MediaManifest; previous?: MediaAssetRecord } {
+  const assets = [...manifest.assets]
+  const index = assets.findIndex((asset) => asset.id === record.id)
+  const previous = index === -1 ? undefined : assets[index]
+  if (index === -1) assets.push(record)
+  else assets[index] = record
+  return {
+    previous,
+    manifest: {
+      topic: manifest.topic,
+      updatedAt: nowIso(),
+      assets,
+    },
+  }
+}
+
+function writeManifest(manifestPath: string, manifest: MediaManifest): void {
+  mkdirSync(dirname(manifestPath), { recursive: true })
+  writeFileSync(manifestPath, `${JSON.stringify(manifest, null, 2)}\n`, "utf-8")
+}
+
+function saveFailureResult(
+  input: MediaSaveInput,
+  workspaceDir: string,
+  topic: string,
+  manifestPath: string,
+  status: Exclude<MediaStatus, "success">,
+  failureReason: string,
+  sourcePath?: string,
+): MediaSaveResult {
+  const manifest = readManifest(manifestPath, topic)
+  const existing = manifest.assets.find((asset) => asset.id === slugify(input.id))
+  const record = existing?.path
+    ? {
+      ...existing,
+      sourceUrl: input.sourceUrl ?? existing.sourceUrl,
+      sourcePath: sourcePath ?? existing.sourcePath,
+      alt: input.alt ?? existing.alt,
+      notes: input.notes ?? existing.notes,
+      failureReason,
+    }
+    : buildFailureRecord(input, topic, status, failureReason, sourcePath)
+  const { manifest: nextManifest, previous } = upsertAsset(manifest, record)
+  writeManifest(manifestPath, nextManifest)
+
+  if (!existing?.path && previous?.path) {
+    rmSync(join(workspaceDir, previous.path), { force: true })
+  }
+
+  return {
+    ok: true,
+    assetId: record.id,
+    status,
+    path: null,
+    manifestPath: relative(workspaceDir, manifestPath),
+    updated: true,
+  }
+}
+
+export async function saveMediaAsset(input: MediaSaveInput, workspaceDir: string): Promise<MediaSaveResult> {
+  try {
+    const topic = slugify(input.topic)
+    const assetId = slugify(input.id)
+    if (!topic) return { ok: false, error: "topic is required" }
+    if (!assetId) return { ok: false, error: "id is required" }
+    if (input.type !== "image") return { ok: false, error: "type must be 'image'" }
+
+    const topicDir = join(workspaceDir, "assets", topic)
+    const mediaDir = join(topicDir, "media")
+    const manifestPath = join(topicDir, "media-manifest.json")
+    mkdirSync(mediaDir, { recursive: true })
+
+    const relativeSourcePath = input.sourcePath
+      ? relative(workspaceDir, ensureInsideWorkspace(input.sourcePath, workspaceDir))
+      : undefined
+
+    if (input.status !== "success") {
+      if (!input.failureReason?.trim()) {
+        return { ok: false, error: "failureReason is required when status is not 'success'" }
+      }
+      return saveFailureResult(
+        input,
+        workspaceDir,
+        topic,
+        manifestPath,
+        input.status,
+        input.failureReason.trim(),
+        relativeSourcePath,
+      )
+    }
+
+    if (!input.sourcePath && !input.sourceUrl) {
+      return { ok: false, error: "sourcePath or sourceUrl is required when status is 'success'" }
+    }
+
+    let buffer: Buffer
+    let extension: string | null = null
+
+    if (input.sourcePath) {
+      const absPath = ensureInsideWorkspace(input.sourcePath, workspaceDir)
+      if (!existsSync(absPath)) {
+        return { ok: false, error: `sourcePath not found: ${input.sourcePath}` }
+      }
+      extension = inferImageExtension(null, absPath)
+      if (!extension) {
+        return { ok: false, error: `unsupported local image type: ${input.sourcePath}` }
+      }
+      const destinationPath = join(mediaDir, `${assetId}${extension}`)
+      const manifest = readManifest(manifestPath, topic)
+      const previous = manifest.assets.find((asset) => asset.id === assetId)
+      copyFileSync(absPath, destinationPath)
+      const record: MediaAssetRecord = {
+        id: assetId,
+        type: "image",
+        purpose: input.purpose,
+        brief: input.brief,
+        status: "success",
+        path: relative(workspaceDir, destinationPath),
+        sourceUrl: input.sourceUrl,
+        sourcePath: relativeSourcePath,
+        intendedSection: input.intendedSection,
+        alt: input.alt,
+        notes: input.notes,
+        savedAt: nowIso(),
+      }
+      const { manifest: nextManifest } = upsertAsset(manifest, record)
+      writeManifest(manifestPath, nextManifest)
+      if (previous?.path && previous.path !== record.path) {
+        rmSync(join(workspaceDir, previous.path), { force: true })
+      }
+      return {
+        ok: true,
+        assetId,
+        status: "success",
+        path: record.path,
+        manifestPath: relative(workspaceDir, manifestPath),
+        updated: true,
+      }
+    }
+
+    try {
+      const downloaded = await downloadImageFromUrl(input.sourceUrl!)
+      buffer = downloaded.buffer
+      extension = downloaded.extension
+    } catch (error) {
+      const message = error instanceof Error ? error.message : String(error)
+      if (message === "INVALID_URL") {
+        return saveFailureResult(input, workspaceDir, topic, manifestPath, "invalid-url", "Invalid image URL")
+      }
+      return saveFailureResult(
+        input,
+        workspaceDir,
+        topic,
+        manifestPath,
+        "cannot-download",
+        `Failed to download image: ${message}`,
+      )
+    }
+
+    const destinationPath = join(mediaDir, `${assetId}${extension}`)
+    const manifest = readManifest(manifestPath, topic)
+    const previous = manifest.assets.find((asset) => asset.id === assetId)
+    writeFileSync(destinationPath, new Uint8Array(buffer))
+    const record: MediaAssetRecord = {
+      id: assetId,
+      type: "image",
+      purpose: input.purpose,
+      brief: input.brief,
+      status: "success",
+      path: relative(workspaceDir, destinationPath),
+      sourceUrl: input.sourceUrl,
+      sourcePath: relativeSourcePath,
+      intendedSection: input.intendedSection,
+      alt: input.alt,
+      notes: input.notes,
+      savedAt: nowIso(),
+    }
+    const { manifest: nextManifest } = upsertAsset(manifest, record)
+    writeManifest(manifestPath, nextManifest)
+    if (previous?.path && previous.path !== record.path) {
+      rmSync(join(workspaceDir, previous.path), { force: true })
+    }
+
+    return {
+      ok: true,
+      assetId,
+      status: "success",
+      path: record.path,
+      manifestPath: relative(workspaceDir, manifestPath),
+      updated: true,
+    }
+  } catch (error) {
+    return {
+      ok: false,
+      error: error instanceof Error ? error.message : String(error),
+    }
+  }
+}

package/lib/media/types.ts

@@ -0,0 +1,54 @@
+export type MediaPurpose = "hero" | "illustration" | "portrait" | "logo" | "screenshot"
+
+export type MediaStatus = "success" | "cannot-download" | "invalid-url" | "cannot-generate"
+
+export interface MediaSaveInput {
+  topic: string
+  id: string
+  type: "image"
+  purpose: MediaPurpose
+  brief: string
+  status: MediaStatus
+  intendedSection?: string
+  sourcePath?: string
+  sourceUrl?: string
+  alt?: string
+  notes?: string
+  failureReason?: string
+}
+
+export interface MediaAssetRecord {
+  id: string
+  type: "image"
+  purpose: MediaPurpose
+  brief: string
+  status: MediaStatus
+  path: string | null
+  sourceUrl?: string
+  sourcePath?: string
+  intendedSection?: string
+  alt?: string
+  notes?: string
+  failureReason?: string
+  savedAt: string
+}
+
+export interface MediaManifest {
+  topic: string
+  updatedAt: string
+  assets: MediaAssetRecord[]
+}
+
+export type MediaSaveResult =
+  | {
+    ok: true
+    assetId: string
+    status: MediaStatus
+    path: string | null
+    manifestPath: string
+    updated: boolean
+  }
+  | {
+    ok: false
+    error: string
+  }

package/lib/research/image-leads.ts

@@ -0,0 +1,175 @@
+import { existsSync, readdirSync, readFileSync } from "fs"
+import { basename, join } from "path"
+
+export type ResearchImageUse = "logo" | "portrait" | "screenshot" | "unknown"
+
+export interface ResearchImageLead {
+  candidateId: string
+  topic: string
+  axis: string
+  sourceFile: string
+  description: string
+  url: string
+  alt: string
+  use: ResearchImageUse
+  line: number
+  valid: boolean
+  warnings: string[]
+}
+
+export interface ListResearchImageLeadOptions {
+  uses?: string[]
+  axis?: string[]
+}
+
+export interface ResearchImageLeadListResult {
+  topic: string
+  items: ResearchImageLead[]
+  warnings: string[]
+}
+
+const LEAD_LINE_RE = /^-\s*(.+?):\s*(\S+)(?:\s*\|\s*Alt:\s*(.*?))?(?:\s*\|\s*Use:\s*(.*?))?\s*$/
+
+function slugify(value: string): string {
+  return value
+    .toLowerCase()
+    .replace(/[^a-z0-9]+/g, "-")
+    .replace(/^-+|-+$/g, "")
+}
+
+function normalizeUse(value: string | undefined): ResearchImageUse {
+  const normalized = (value ?? "").trim().toLowerCase()
+  if (normalized === "logo" || normalized === "portrait" || normalized === "screenshot") {
+    return normalized
+  }
+  return "unknown"
+}
+
+function normalizeAxisFilter(values: string[] | undefined): string[] | undefined {
+  if (!values?.length) return undefined
+  const normalized = values
+    .map((value) => slugify(value))
+    .filter(Boolean)
+  return normalized.length ? normalized : undefined
+}
+
+function normalizeUseFilter(values: string[] | undefined): ResearchImageUse[] | undefined {
+  if (!values?.length) return undefined
+  const normalized = values
+    .map((value) => normalizeUse(value))
+  return normalized.length ? normalized : undefined
+}
+
+function isHttpUrl(value: string): boolean {
+  try {
+    const url = new URL(value)
+    return url.protocol === "http:" || url.protocol === "https:"
+  } catch {
+    return false
+  }
+}
+
+function shouldIncludeLead(lead: ResearchImageLead, options?: ListResearchImageLeadOptions): boolean {
+  const uses = normalizeUseFilter(options?.uses)
+  const axis = normalizeAxisFilter(options?.axis)
+  if (uses?.length && !uses.includes(lead.use)) return false
+  if (axis?.length && !axis.includes(lead.axis)) return false
+  return true
+}
+
+function parseImageSection(
+  topic: string,
+  axis: string,
+  sourceFile: string,
+  content: string,
+): { items: ResearchImageLead[]; warnings: string[] } {
+  const lines = content.split(/\r?\n/)
+  const items: ResearchImageLead[] = []
+  const warnings: string[] = []
+  let inImagesSection = false
+  let candidateIndex = 0
+
+  for (let index = 0; index < lines.length; index++) {
+    const line = lines[index]
+    const trimmed = line.trim()
+
+    if (/^##\s+Images\s*$/i.test(trimmed)) {
+      inImagesSection = true
+      continue
+    }
+
+    if (inImagesSection && /^##\s+/.test(trimmed)) {
+      break
+    }
+
+    if (!inImagesSection || !trimmed.startsWith("-")) continue
+    if (trimmed === "- None") continue
+
+    const match = trimmed.match(LEAD_LINE_RE)
+    if (!match) {
+      warnings.push(`${sourceFile}:${index + 1} could not parse image lead`)
+      continue
+    }
+
+    candidateIndex += 1
+    const description = match[1].trim()
+    const url = match[2].trim()
+    const alt = (match[3] ?? "").trim()
+    const use = normalizeUse(match[4])
+    const itemWarnings: string[] = []
+    const valid = isHttpUrl(url)
+
+    if (!valid) itemWarnings.push("invalid-url")
+    if (use === "unknown") itemWarnings.push("unknown-use")
+
+    items.push({
+      candidateId: `${axis}:${candidateIndex}`,
+      topic,
+      axis,
+      sourceFile,
+      description,
+      url,
+      alt,
+      use,
+      line: index + 1,
+      valid,
+      warnings: itemWarnings,
+    })
+  }
+
+  return { items, warnings }
+}
+
+export function listResearchImageLeads(
+  topic: string,
+  workspaceDir: string,
+  options?: ListResearchImageLeadOptions,
+): ResearchImageLeadListResult {
+  const topicSlug = slugify(topic)
+  const researchDir = join(workspaceDir, "researches", topicSlug)
+  if (!existsSync(researchDir)) {
+    return { topic: topicSlug, items: [], warnings: [] }
+  }
+
+  const items: ResearchImageLead[] = []
+  const warnings: string[] = []
+  const entries = readdirSync(researchDir)
+    .filter((entry) => entry.endsWith(".md"))
+    .sort()
+
+  for (const entry of entries) {
+    const filePath = join(researchDir, entry)
+    const axis = basename(entry, ".md")
+    const sourceFile = `researches/${topicSlug}/${entry}`
+    const content = readFileSync(filePath, "utf-8")
+    const parsed = parseImageSection(topicSlug, axis, sourceFile, content)
+
+    for (const item of parsed.items) {
+      if (shouldIncludeLead(item, options)) items.push(item)
+    }
+
+    warnings.push(...parsed.warnings)
+  }
+
+  return { topic: topicSlug, items, warnings }
+}