ima2-gen 1.1.7 → 1.1.8

package/lib/promptImport/parsePromptCandidates.js

@@ -29,6 +29,10 @@ function candidateId(text, ordinal) {
   return `candidate_${ordinal}_${createHash("sha256").update(text).digest("hex").slice(0, 10)}`;
 }
 
+function promptHash(text) {
+  return createHash("sha256").update(normalizeWhitespace(text).toLowerCase()).digest("hex");
+}
+
 function headingName(heading, fallback) {
   return heading?.replace(/^#+\s*/, "").trim() || fallback;
 }
@@ -52,13 +56,19 @@ function pushCandidate(candidates, rawText, options) {
   const text = normalizeWhitespace(rawText);
   if (!allowedCandidate(text, options.limits)) return;
   const ordinal = candidates.length + 1;
+  const hash = promptHash(text);
   candidates.push({
     id: candidateId(text, ordinal),
     name: options.name,
     text,
+    textPreview: text.slice(0, 220),
     tags: [...new Set(options.tags)],
     warnings: options.warnings ?? [],
     source: options.source,
+    headingPath: options.headingPath ?? null,
+    ordinal,
+    promptHash: hash,
+    scoreHints: options.scoreHints ?? {},
   });
 }
 
@@ -75,6 +85,7 @@ function parseMarkdown(text, options) {
     pushCandidate(options.candidates, match[2], {
       ...options,
       name: `${options.baseName} ${options.candidates.length + 1}`,
+      headingPath: options.headingPath ?? null,
     });
     if (options.candidates.length >= options.limits.maxPromptCandidatesPerFile) return;
   }
@@ -93,6 +104,7 @@ function parseMarkdown(text, options) {
     pushCandidate(options.candidates, body, {
       ...options,
       name: headingName(heading, `${options.baseName} ${options.candidates.length + 1}`),
+      headingPath: headingName(heading, options.baseName),
     });
     if (options.candidates.length >= options.limits.maxPromptCandidatesPerFile) return;
   }
@@ -116,6 +128,7 @@ function parsePlainText(text, options) {
     pushCandidate(options.candidates, clean, {
       ...options,
       name: `${options.baseName} ${options.candidates.length + 1}`,
+      headingPath: `${options.baseName} ${options.candidates.length + 1}`,
     });
     if (options.candidates.length >= options.limits.maxPromptCandidatesPerFile) return;
   }

package/lib/promptImport/promptIndex.js

@@ -0,0 +1,248 @@
+import { createHash } from "node:crypto";
+import { mkdir, readFile, rename, writeFile } from "node:fs/promises";
+import { dirname } from "node:path";
+import { getCuratedSource, getDefaultSearchSources, listCuratedSources } from "./curatedSources.js";
+import { buildGitHubRawFileSource, fetchGitHubSource } from "./githubSource.js";
+import { parsePromptCandidates } from "./parsePromptCandidates.js";
+import { extractGptImageHints } from "./gptImageHints.js";
+import { rankPromptCandidates } from "./rankPromptCandidates.js";
+import {
+  getDefaultReviewedDiscoverySources,
+  getReviewedDiscoverySource,
+  listReviewedDiscoverySources,
+} from "./discoveryRegistry.js";
+
+const INDEX_VERSION = 1;
+const EXTRACTOR_VERSION = 2;
+
+function limitsFromCtx(ctx) {
+  return {
+    maxFileBytesForPreview: ctx.config.limits.promptImportMaxFileBytes,
+    maxPromptCandidatesPerFile: ctx.config.limits.promptImportMaxCandidatesPerFile,
+    maxPromptCandidatesPerImport: ctx.config.limits.promptImportMaxCandidatesPerImport,
+    fetchTimeoutMs: ctx.config.limits.promptImportFetchTimeoutMs,
+    maxCandidateChars: ctx.config.limits.promptImportMaxCandidateChars,
+    minCandidateChars: ctx.config.limits.promptImportMinCandidateChars,
+    maxSourceCharsScanned: ctx.config.limits.promptImportMaxSourceCharsScanned,
+    maxRepoIndexFiles: ctx.config.limits.promptImportMaxRepoIndexFiles,
+    searchLimit: ctx.config.limits.promptImportCuratedSearchLimit,
+    ttlMs: ctx.config.limits.promptImportIndexCacheTtlMs,
+  };
+}
+
+function cacheFile(ctx) {
+  return ctx.config.storage.promptImportIndexCacheFile;
+}
+
+function sourceFileId(source, path) {
+  return `github:${source.repo}@${source.defaultRef}:${path}`;
+}
+
+function hashId(...parts) {
+  return createHash("sha256").update(parts.join("\0")).digest("hex");
+}
+
+async function readCache(ctx) {
+  try {
+    const parsed = JSON.parse(await readFile(cacheFile(ctx), "utf8"));
+    if (parsed.version !== INDEX_VERSION) return { version: INDEX_VERSION, sources: {} };
+    return { version: INDEX_VERSION, sources: parsed.sources || {} };
+  } catch {
+    return { version: INDEX_VERSION, sources: {} };
+  }
+}
+
+async function writeCache(ctx, cache) {
+  const file = cacheFile(ctx);
+  await mkdir(dirname(file), { recursive: true });
+  const tmp = `${file}.${process.pid}.${Date.now()}.tmp`;
+  await writeFile(tmp, JSON.stringify(cache, null, 2));
+  await rename(tmp, file);
+}
+
+function sourceTags(source, fileSource) {
+  return [
+    ...fileSource.tags,
+    `source:${source.id}`,
+    `license:${source.licenseSpdx}`,
+    `trust:${source.trustTier}`,
+    source.requiresAttribution ? "attribution-required" : null,
+  ].filter(Boolean);
+}
+
+function indexedCandidate({ candidate, source, fileSource, fileIndex, index }) {
+  const scoreHints = extractGptImageHints(candidate.text);
+  const headingPath = candidate.headingPath || candidate.name || "";
+  const candidateId = hashId(fileIndex.sourceFileId, fileIndex.contentHash, headingPath, String(candidate.ordinal || index + 1));
+  const tags = [...new Set([...(candidate.tags || []), ...sourceTags(source, fileSource)])];
+  return {
+    ...candidate,
+    id: candidateId,
+    candidateId,
+    name: candidate.name,
+    text: candidate.text,
+    textPreview: candidate.textPreview || candidate.text.slice(0, 220),
+    tags,
+    warnings: [...new Set([...(candidate.warnings || []), ...scoreHints.warnings])],
+    source: {
+      kind: "github",
+      owner: source.owner,
+      repo: source.name,
+      ref: source.defaultRef,
+      path: fileSource.path,
+      htmlUrl: fileSource.htmlUrl,
+      sourceId: source.id,
+    },
+    sourceFileId: fileIndex.sourceFileId,
+    headingPath,
+    ordinal: candidate.ordinal || index + 1,
+    promptHash: candidate.promptHash || hashId(candidate.text.trim().toLowerCase()),
+    scoreHints,
+  };
+}
+
+async function indexSource(ctx, sourceId) {
+  const source = getCuratedSource(sourceId) || await getReviewedDiscoverySource(ctx, sourceId);
+  if (!source || source.trustTier === "manual-review") {
+    return { source, indexedFiles: 0, candidateCount: 0, warnings: ["curated-source-unavailable"] };
+  }
+  if (String(source.defaultRef || "").includes("/")) {
+    return { source, indexedFiles: 0, candidateCount: 0, warnings: ["discovery-default-branch-unsupported"] };
+  }
+  if (!Array.isArray(source.allowedPaths) || source.allowedPaths.length === 0) {
+    return { source, indexedFiles: 0, candidateCount: 0, warnings: ["discovery-requires-paths"] };
+  }
+
+  const limits = limitsFromCtx(ctx);
+  const warnings = [];
+  const files = [];
+  const candidates = [];
+  const allowedPaths = source.allowedPaths.slice(0, limits.maxRepoIndexFiles);
+
+  for (const path of allowedPaths) {
+    try {
+      const fileSource = buildGitHubRawFileSource({
+        owner: source.owner,
+        repo: source.name,
+        ref: source.defaultRef,
+        path,
+      });
+      const fetched = await fetchGitHubSource(fileSource, limits);
+      const fileIndex = {
+        sourceFileId: sourceFileId(source, path),
+        owner: source.owner,
+        repo: source.name,
+        ref: source.defaultRef,
+        path,
+        extension: fileSource.extension,
+        contentHash: fetched.contentHash,
+        etag: fetched.etag,
+        sizeBytes: fetched.sizeBytes,
+        licenseSpdx: source.licenseSpdx,
+        htmlUrl: fileSource.htmlUrl,
+        indexedAt: new Date().toISOString(),
+        lastFetchStatus: "ok",
+        promptCandidateCount: 0,
+        extractorVersion: EXTRACTOR_VERSION,
+      };
+      const parsed = parsePromptCandidates({
+        text: fetched.text,
+        filename: path,
+        source: { kind: "github", owner: source.owner, repo: source.name, ref: source.defaultRef, path, htmlUrl: fileSource.htmlUrl },
+        tags: sourceTags(source, fileSource),
+        limits,
+      });
+      const indexed = parsed.map((candidate, index) => indexedCandidate({ candidate, source, fileSource, fileIndex, index }));
+      fileIndex.promptCandidateCount = indexed.length;
+      files.push(fileIndex);
+      candidates.push(...indexed);
+    } catch (error) {
+      warnings.push(`${path}: ${error?.message || "index failed"}`);
+    }
+  }
+
+  return {
+    source,
+    indexedFiles: files.length,
+    candidateCount: candidates.length,
+    warnings,
+    entry: {
+      source,
+      files,
+      candidates,
+      refreshedAt: Date.now(),
+    },
+  };
+}
+
+function isFresh(entry, ttlMs) {
+  return entry?.refreshedAt && Date.now() - entry.refreshedAt < ttlMs;
+}
+
+async function ensureSearchCache(ctx) {
+  const cache = await readCache(ctx);
+  const limits = limitsFromCtx(ctx);
+  const sources = [
+    ...getDefaultSearchSources(),
+    ...await getDefaultReviewedDiscoverySources(ctx),
+  ];
+  let changed = false;
+  const warnings = [];
+
+  for (const source of sources) {
+    if (isFresh(cache.sources[source.id], limits.ttlMs)) continue;
+    const result = await indexSource(ctx, source.id);
+    if (result.entry) {
+      cache.sources[source.id] = result.entry;
+      changed = true;
+    }
+    warnings.push(...result.warnings);
+  }
+  if (changed) await writeCache(ctx, cache);
+  return { cache, warnings };
+}
+
+export async function refreshCuratedSource(ctx, sourceId) {
+  const cache = await readCache(ctx);
+  const result = await indexSource(ctx, sourceId);
+  if (result.entry) {
+    cache.sources[sourceId] = result.entry;
+    await writeCache(ctx, cache);
+  }
+  return {
+    source: result.source,
+    indexedFiles: result.indexedFiles,
+    candidateCount: result.candidateCount,
+    warnings: result.warnings,
+  };
+}
+
+export async function searchCuratedPrompts(ctx, { q = "", sourceIds, limit } = {}) {
+  const { cache, warnings } = await ensureSearchCache(ctx);
+  const limits = limitsFromCtx(ctx);
+  const defaultSources = [
+    ...getDefaultSearchSources(),
+    ...await getDefaultReviewedDiscoverySources(ctx),
+  ];
+  const allowedIds = Array.isArray(sourceIds) && sourceIds.length
+    ? new Set(sourceIds)
+    : new Set(defaultSources.map((source) => source.id));
+  const candidates = Object.values(cache.sources)
+    .filter((entry) => allowedIds.has(entry.source.id))
+    .flatMap((entry) => entry.candidates || []);
+  const results = rankPromptCandidates({
+    candidates,
+    query: q,
+    limit: Math.min(Number(limit) || limits.searchLimit, limits.searchLimit),
+  });
+  const sources = [
+    ...listCuratedSources(),
+    ...await listReviewedDiscoverySources(ctx),
+  ];
+  return { results, sources, warnings };
+}
+
+export async function getPromptImportSources(ctx) {
+  const reviewed = ctx ? await listReviewedDiscoverySources(ctx) : [];
+  return { sources: [...listCuratedSources(), ...reviewed] };
+}

package/lib/promptImport/rankPromptCandidates.js

@@ -0,0 +1,49 @@
+function tokenize(value) {
+  return String(value || "")
+    .toLowerCase()
+    .split(/[^a-z0-9가-힣-]+/i)
+    .map((token) => token.trim())
+    .filter(Boolean);
+}
+
+function includesAny(values, terms) {
+  const haystack = values.map((value) => String(value || "").toLowerCase());
+  return terms.some((term) => haystack.some((value) => value.includes(term)));
+}
+
+export function rankPromptCandidates({ candidates, query, limit }) {
+  const terms = tokenize(query);
+  const boundedLimit = Math.max(1, Math.min(Number(limit) || 50, 100));
+  const ranked = candidates.map((candidate) => {
+    const text = String(candidate.text || "").toLowerCase();
+    const name = String(candidate.name || "").toLowerCase();
+    const tags = Array.isArray(candidate.tags) ? candidate.tags : [];
+    const hints = candidate.scoreHints || {};
+    const hintValues = [
+      ...(hints.modelHints || []),
+      ...(hints.generationSurfaceHints || []),
+      ...(hints.taskHints || []),
+      ...(hints.sizeHints || []),
+      ...(hints.qualityHints || []),
+    ];
+    let score = 0;
+
+    if (terms.length === 0) score += 1;
+    for (const term of terms) {
+      if (name.includes(term)) score += 18;
+      if (includesAny(tags, [term])) score += 12;
+      if (includesAny(hintValues, [term])) score += 14;
+      if (text.includes(term)) score += 5;
+      if (String(candidate.sourceFileId || "").toLowerCase().includes(term)) score += 4;
+    }
+    if (tags.some((tag) => tag === "trust:curated")) score += 5;
+    if (candidate.warnings?.length) score -= candidate.warnings.length * 3;
+
+    return { ...candidate, score };
+  });
+
+  return ranked
+    .filter((candidate) => terms.length === 0 || candidate.score > 0)
+    .sort((a, b) => b.score - a.score || String(a.name).localeCompare(String(b.name)))
+    .slice(0, boundedLimit);
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ima2-gen",
-  "version": "1.1.7",
+  "version": "1.1.8",
   "description": "Local OAuth image generation studio with classic and node workflows",
   "type": "module",
   "bin": {

package/routes/imageImport.js

@@ -0,0 +1,33 @@
+import express from "express";
+import { createLocalImport } from "../lib/localImportStore.js";
+
+function decodeHeader(value) {
+  if (typeof value !== "string" || !value) return null;
+  try {
+    return decodeURIComponent(value);
+  } catch {
+    return value;
+  }
+}
+
+export function registerImageImportRoutes(app, ctx) {
+  const rawImage = express.raw({
+    type: ["image/png", "image/jpeg", "image/webp"],
+    limit: ctx.config.server.bodyLimit,
+  });
+
+  app.post("/api/history/import-local", rawImage, async (req, res) => {
+    try {
+      const item = await createLocalImport(ctx, {
+        buffer: Buffer.isBuffer(req.body) ? req.body : Buffer.alloc(0),
+        originalFilename: decodeHeader(req.headers["x-ima2-original-filename"]),
+      });
+      res.status(201).json({ item });
+    } catch (err) {
+      res.status(err.status || 500).json({
+        error: err.message,
+        code: err.code || "IMPORT_FAILED",
+      });
+    }
+  });
+}

package/routes/index.js

@@ -13,6 +13,7 @@ import { registerPromptImportRoutes } from "./promptImport.js";
 import { registerAnnotationRoutes } from "./annotations.js";
 import { registerCanvasVersionRoutes } from "./canvasVersions.js";
 import { registerComfyRoutes } from "./comfy.js";
+import { registerImageImportRoutes } from "./imageImport.js";
 
 export function configureRoutes(app, ctx) {
   registerHealthRoutes(app, ctx);
@@ -21,6 +22,7 @@ export function configureRoutes(app, ctx) {
   registerHistoryRoutes(app, ctx);
   registerAnnotationRoutes(app, ctx);
   registerCanvasVersionRoutes(app, ctx);
+  registerImageImportRoutes(app, ctx);
   registerComfyRoutes(app, ctx);
   registerSessionRoutes(app, ctx);
   registerEditRoutes(app, ctx);

package/routes/promptImport.js

@@ -6,7 +6,22 @@ import {
   isSupportedPromptFileName,
   normalizeGitHubSource,
 } from "../lib/promptImport/githubSource.js";
+import {
+  fetchGitHubFolderFiles,
+  fetchSelectedGitHubFolderFiles,
+  normalizeGitHubFolderSource,
+} from "../lib/promptImport/githubFolder.js";
 import { parsePromptCandidates } from "../lib/promptImport/parsePromptCandidates.js";
+import {
+  getPromptImportSources,
+  refreshCuratedSource,
+  searchCuratedPrompts,
+} from "../lib/promptImport/promptIndex.js";
+import { searchGitHubDiscovery } from "../lib/promptImport/githubDiscovery.js";
+import {
+  listDiscoveryCandidates,
+  reviewDiscoveryCandidate,
+} from "../lib/promptImport/discoveryRegistry.js";
 
 function promptImportLimits(ctx) {
   return {
@@ -17,6 +32,13 @@ function promptImportLimits(ctx) {
     maxCandidateChars: ctx.config.limits.promptImportMaxCandidateChars,
     minCandidateChars: ctx.config.limits.promptImportMinCandidateChars,
     maxSourceCharsScanned: ctx.config.limits.promptImportMaxSourceCharsScanned,
+    maxRepoIndexFiles: ctx.config.limits.promptImportMaxRepoIndexFiles,
+    curatedSearchLimit: ctx.config.limits.promptImportCuratedSearchLimit,
+    indexCacheTtlMs: ctx.config.limits.promptImportIndexCacheTtlMs,
+    maxFolderFiles: ctx.config.limits.promptImportMaxFolderFiles,
+    maxFolderPreviewFiles: ctx.config.limits.promptImportMaxFolderPreviewFiles,
+    discoverySearchLimit: ctx.config.limits.promptImportDiscoverySearchLimit,
+    discoveryMaxQueries: ctx.config.limits.promptImportDiscoveryMaxQueries,
   };
 }
 
@@ -96,6 +118,58 @@ async function buildPreview(req, ctx) {
   return { source, candidates, warnings: [] };
 }
 
+function normalizeFolderInput(body) {
+  const input = typeof body?.source?.input === "string" ? body.source.input : body?.input;
+  return normalizeGitHubFolderSource(input);
+}
+
+async function buildFolderFiles(req, ctx) {
+  const limits = promptImportLimits(ctx);
+  const source = normalizeFolderInput(req.body || {});
+  return fetchGitHubFolderFiles(source, limits);
+}
+
+async function buildFolderPreview(req, ctx) {
+  const limits = promptImportLimits(ctx);
+  const source = normalizeFolderInput(req.body || {});
+  const paths = Array.isArray(req.body?.paths) ? req.body.paths : [];
+  const selected = await fetchSelectedGitHubFolderFiles(source, paths, limits);
+  const candidates = [];
+  const warnings = [...selected.warnings];
+
+  for (const file of selected.files) {
+    const text = file.text.length > limits.maxSourceCharsScanned
+      ? file.text.slice(0, limits.maxSourceCharsScanned)
+      : file.text;
+    const parsed = parsePromptCandidates({
+      text,
+      filename: file.path,
+      source: {
+        kind: "github",
+        owner: source.owner,
+        repo: source.repo,
+        ref: source.ref,
+        path: file.path,
+        htmlUrl: file.htmlUrl,
+      },
+      tags: [...source.tags, `file:${file.name}`, `ext:${file.extension}`],
+      limits,
+    });
+    if (parsed.length === 0) warnings.push(`${file.path}: no prompt candidates`);
+    candidates.push(...parsed);
+  }
+
+  if (candidates.length === 0) {
+    throw promptImportError("PROMPT_IMPORT_EMPTY", "No prompt candidates were found", 422);
+  }
+  return {
+    source,
+    files: selected.files.map(({ text, contentHash, ...file }) => file),
+    candidates: candidates.slice(0, limits.maxPromptCandidatesPerImport),
+    warnings,
+  };
+}
+
 function assertCommitCandidateText(text, limits) {
   if (text.length < limits.minCandidateChars) {
     throw promptImportError("PROMPT_IMPORT_EMPTY", "Prompt candidate is too short", 422);
@@ -144,6 +218,111 @@ function commitCandidates(candidates, folderId, limits) {
 }
 
 export function registerPromptImportRoutes(app, ctx) {
+  app.get("/api/prompts/import/curated-sources", async (req, res) => {
+    try {
+      res.json(await getPromptImportSources(ctx));
+    } catch (error) {
+      logError("promptImport", "curated_sources_error", error);
+      sendPromptImportError(res, error);
+    }
+  });
+
+  app.get("/api/prompts/import/discovery", async (req, res) => {
+    try {
+      const candidates = await listDiscoveryCandidates(ctx, {
+        status: typeof req.query?.status === "string" ? req.query.status : undefined,
+      });
+      res.json({ candidates, warnings: [] });
+    } catch (error) {
+      logError("promptImport", "discovery_list_error", error);
+      sendPromptImportError(res, error);
+    }
+  });
+
+  app.post("/api/prompts/import/discovery-search", async (req, res) => {
+    try {
+      const limits = promptImportLimits(ctx);
+      const seeds = Array.isArray(req.body?.seeds) ? req.body.seeds : [];
+      if (seeds.length > limits.discoveryMaxQueries) {
+        throw promptImportError("GITHUB_DISCOVERY_TOO_MANY_QUERIES", "Too many discovery queries", 413);
+      }
+      const result = await searchGitHubDiscovery(ctx, {
+        q: req.body?.q,
+        seeds,
+        limit: req.body?.limit,
+        maxQueries: limits.discoveryMaxQueries,
+      });
+      res.json(result);
+    } catch (error) {
+      logError("promptImport", "discovery_search_error", error);
+      sendPromptImportError(res, error);
+    }
+  });
+
+  app.post("/api/prompts/import/discovery-review", async (req, res) => {
+    try {
+      const result = await reviewDiscoveryCandidate(ctx, {
+        repo: req.body?.repo,
+        status: req.body?.status,
+        reviewNotes: req.body?.reviewNotes,
+        allowedPaths: req.body?.allowedPaths,
+        defaultSearch: req.body?.defaultSearch,
+      });
+      res.json(result);
+    } catch (error) {
+      logError("promptImport", "discovery_review_error", error);
+      sendPromptImportError(res, error);
+    }
+  });
+
+  app.post("/api/prompts/import/curated-search", async (req, res) => {
+    try {
+      const result = await searchCuratedPrompts(ctx, {
+        q: req.body?.q,
+        sourceIds: req.body?.sourceIds,
+        limit: req.body?.limit,
+      });
+      res.json(result);
+    } catch (error) {
+      logError("promptImport", "curated_search_error", error);
+      sendPromptImportError(res, error);
+    }
+  });
+
+  app.post("/api/prompts/import/curated-refresh", async (req, res) => {
+    try {
+      const sourceId = typeof req.body?.sourceId === "string" ? req.body.sourceId : "";
+      if (!sourceId) {
+        throw promptImportError("INVALID_GITHUB_SOURCE", "Curated source is required", 400);
+      }
+      const result = await refreshCuratedSource(ctx, sourceId);
+      res.json(result);
+    } catch (error) {
+      logError("promptImport", "curated_refresh_error", error);
+      sendPromptImportError(res, error);
+    }
+  });
+
+  app.post("/api/prompts/import/folder-files", async (req, res) => {
+    try {
+      const result = await buildFolderFiles(req, ctx);
+      res.json(result);
+    } catch (error) {
+      logError("promptImport", "folder_files_error", error);
+      sendPromptImportError(res, error);
+    }
+  });
+
+  app.post("/api/prompts/import/folder-preview", async (req, res) => {
+    try {
+      const result = await buildFolderPreview(req, ctx);
+      res.json(result);
+    } catch (error) {
+      logError("promptImport", "folder_preview_error", error);
+      sendPromptImportError(res, error);
+    }
+  });
+
   app.post("/api/prompts/import/preview", async (req, res) => {
     try {
       const preview = await buildPreview(req, ctx);