@tuongaz/seeflow 0.1.98 → 0.1.99

package/src/cli.ts

@@ -196,6 +196,8 @@ if (argv.includes('--version') || argv.includes('-v')) {
   await runFlowsDelete();
 } else if (sub === 'flows:layout') {
   await runFlowsLayout();
+} else if (sub === 'icons') {
+  await runIcons();
 } else if (sub === 'flow:add-bulk') {
   await runFlowAddBulk();
 } else if (sub === 'flows:play') {
@@ -286,6 +288,12 @@ Commands (work without a running studio):
                        when seeding a flow.json (e.g. 'ids node 10', then
                        'ids connector 12').
 
+Icons (local cache):
+  icons list           List installed and available vendor packs (aws, azure)
+  icons add <v>        Install a vendor pack (v: aws|azure) [--accept-terms] [--pack-url <url>]
+  icons update <v>     Re-install a vendor pack — same flags as add
+  icons remove <v>     Remove an installed vendor pack (idempotent)
+
 Commands (require a running studio):
   flows:play <n>       Trigger a play on node <n> (--project <p> --flow <f>)
   emit <id> <n> <st>   Broadcast a status event for node <n> (st: running|done|error)
@@ -1218,3 +1226,108 @@ async function runE2e() {
   }
   printOk(report);
 }
+
+async function runIcons() {
+  const action = argv[1];
+  switch (action) {
+    case undefined:
+    case 'list':
+      await runIconsList();
+      break;
+    case 'add':
+      await runIconsAdd();
+      break;
+    case 'update':
+      await runIconsUpdate();
+      break;
+    case 'remove':
+      await runIconsRemove();
+      break;
+    default:
+      console.error(`Unknown icons action: ${action}`);
+      console.error('Usage: seeflow icons {list|add|update|remove} ...');
+      process.exit(1);
+  }
+}
+
+async function runIconsList() {
+  const { iconCacheRoot } = await import('./icons/paths.ts');
+  const { readIndex } = await import('./icons/index-store.ts');
+  const { summarizePacks } = await import('./icons/list-helper.ts');
+  const idx = readIndex(iconCacheRoot());
+  printOk({ packs: summarizePacks(idx) });
+}
+
+type IconVendorSlug = 'aws' | 'azure';
+
+function parseIconVendor(action: 'add' | 'remove'): IconVendorSlug {
+  const vendors: readonly IconVendorSlug[] = ['aws', 'azure'];
+  const raw = argv[2];
+  if (!raw || raw.startsWith('--')) {
+    const suffix = action === 'add' ? ' [--accept-terms] [--pack-url <url>]' : '';
+    console.error(`Usage: seeflow icons ${action} <vendor>${suffix}`);
+    process.exit(1);
+  }
+  if (!(vendors as readonly string[]).includes(raw)) {
+    console.error(`Unknown vendor: ${raw}. Expected one of: ${vendors.join(', ')}.`);
+    process.exit(1);
+  }
+  return raw as IconVendorSlug;
+}
+
+async function runIconsAdd() {
+  const vendor = parseIconVendor('add');
+  const acceptTerms = hasFlag('accept-terms');
+  const packUrl = flagValue('pack-url');
+
+  const { installIconPack } = await import('./icons/installer.ts');
+  const { fetchWithProgress } = await import('./icons/fetcher.ts');
+  const { iconCacheRoot } = await import('./icons/paths.ts');
+
+  const gen = installIconPack(
+    { vendor, acceptTerms, packUrl },
+    {
+      cacheRoot: iconCacheRoot(),
+      now: () => Date.now(),
+      version: () => new Date().toISOString().slice(0, 10),
+      fetcher: (url: string) =>
+        fetchWithProgress(url, {
+          onProgress: (bytes) => process.stderr.write(`download-progress ${vendor} ${bytes}\n`),
+        }),
+    },
+  );
+
+  for await (const ev of gen) {
+    if (ev.type === 'download-started') {
+      process.stderr.write(`download-started ${ev.vendor}\n`);
+    } else if (ev.type === 'download-progress') {
+      process.stderr.write(`download-progress ${ev.vendor} ${ev.receivedBytes}\n`);
+    } else if (ev.type === 'extracting') {
+      process.stderr.write(`extracting ${ev.vendor}\n`);
+    } else if (ev.type === 'indexing') {
+      process.stderr.write(`indexing ${ev.vendor} (${ev.iconCount} icons)\n`);
+    } else if (ev.type === 'terms-required') {
+      printError(
+        `Vendor '${ev.vendor}' requires accepting the license at ${ev.licenseUrl}. Re-run with --accept-terms.`,
+      );
+    } else if (ev.type === 'error') {
+      printError(`Install failed: ${ev.message}`);
+    } else if (ev.type === 'done') {
+      printOk({ installed: ev.vendor, version: ev.version, iconCount: ev.iconCount });
+    }
+  }
+  printError('Install ended without a terminal event.');
+}
+
+async function runIconsUpdate() {
+  // Installer's rmSync-then-extract handles wipe; behaviour matches add.
+  await runIconsAdd();
+}
+
+async function runIconsRemove() {
+  const vendor = parseIconVendor('remove');
+  const { removeIconPack } = await import('./icons/remove.ts');
+  const { iconCacheRoot } = await import('./icons/paths.ts');
+  removeIconPack(vendor, { cacheRoot: iconCacheRoot() });
+  printOk({ removed: vendor });
+}

package/src/icons/extract-zip.ts

@@ -0,0 +1,31 @@
+import { mkdirSync, writeFileSync } from 'node:fs';
+import { dirname, join, normalize, sep } from 'node:path';
+import { unzipSync } from 'fflate';
+
+export async function extractZipToDir(buffer: Buffer, destDir: string): Promise<string[]> {
+  const entries = unzipSync(new Uint8Array(buffer));
+  const written: string[] = [];
+  const root = normalize(destDir) + sep;
+  for (const [entryPath, data] of Object.entries(entries)) {
+    if (!entryPath.toLowerCase().endsWith('.svg')) continue;
+    const segments = entryPath.split(/[\\/]/);
+    if (segments.some((s) => s === '..')) {
+      throw new Error(`Zip entry escapes destination: ${entryPath}`);
+    }
+    // Skip macOS AppleDouble metadata. The AWS zip ships these alongside the
+    // real SVGs (in __MACOSX/ and as ._-prefixed siblings); they have an .svg
+    // extension but the contents are binary, which renders as broken icons
+    // and pollutes the index with arch-amazon-* duplicate entries.
+    if (segments.includes('__MACOSX')) continue;
+    const flatName = segments.pop();
+    if (!flatName || flatName.startsWith('._')) continue;
+    const target = normalize(join(destDir, flatName));
+    if (!target.startsWith(root)) {
+      throw new Error(`Zip entry escapes destination: ${entryPath}`);
+    }
+    mkdirSync(dirname(target), { recursive: true });
+    writeFileSync(target, data);
+    written.push(flatName);
+  }
+  return written;
+}

package/src/icons/fetcher.ts

@@ -0,0 +1,31 @@
+export interface FetchWithProgressOptions {
+  onProgress?: (receivedBytes: number) => void;
+  fetchFn?: (url: string) => Promise<Response>;
+}
+
+export async function fetchWithProgress(
+  url: string,
+  opts: FetchWithProgressOptions = {},
+): Promise<Buffer> {
+  const fetchFn = opts.fetchFn ?? ((u: string) => fetch(u));
+  const res = await fetchFn(url);
+  if (!res.ok) {
+    throw new Error(`fetch ${url} failed: ${res.status} ${res.statusText}`);
+  }
+  if (!res.body) {
+    throw new Error(`fetch ${url} returned no body`);
+  }
+
+  const reader = res.body.getReader();
+  const chunks: Uint8Array[] = [];
+  let received = 0;
+  while (true) {
+    const { done, value } = await reader.read();
+    if (done) break;
+    if (!value) continue;
+    chunks.push(value);
+    received += value.byteLength;
+    opts.onProgress?.(received);
+  }
+  return Buffer.concat(chunks);
+}

package/src/icons/index-store.ts

@@ -0,0 +1,45 @@
+import { existsSync, readFileSync } from 'node:fs';
+import { join } from 'node:path';
+import { writeFileAtomic } from '../atomic-write.ts';
+import type { IconVendor } from './paths.ts';
+
+export interface InstalledPack {
+  vendor: IconVendor;
+  version: string;
+  installedAt: number;
+  sizeBytes: number;
+  /** Map of canonical icon name → cache-root-relative SVG path. */
+  icons: Record<string, string>;
+}
+
+export interface IconIndex {
+  version: 1;
+  packs: Partial<Record<IconVendor, InstalledPack>>;
+}
+
+const EMPTY: IconIndex = { version: 1, packs: {} };
+
+export function readIndex(cacheRoot: string): IconIndex {
+  const file = join(cacheRoot, 'index.json');
+  if (!existsSync(file)) return { ...EMPTY };
+  try {
+    const parsed = JSON.parse(readFileSync(file, 'utf8')) as IconIndex;
+    if (parsed?.version !== 1 || typeof parsed.packs !== 'object') return { ...EMPTY };
+    return parsed;
+  } catch {
+    return { ...EMPTY };
+  }
+}
+
+export function writeIndex(cacheRoot: string, idx: IconIndex): void {
+  writeFileAtomic(join(cacheRoot, 'index.json'), JSON.stringify(idx, null, 2));
+}
+
+export function upsertPack(idx: IconIndex, pack: InstalledPack): IconIndex {
+  return { version: 1, packs: { ...idx.packs, [pack.vendor]: pack } };
+}
+
+export function removePack(idx: IconIndex, vendor: IconVendor): IconIndex {
+  const { [vendor]: _omit, ...rest } = idx.packs;
+  return { version: 1, packs: rest };
+}

package/src/icons/installer-types.ts

@@ -0,0 +1,16 @@
+import type { IconVendor } from './paths.ts';
+
+export type InstallEvent =
+  | { type: 'terms-required'; vendor: IconVendor; licenseUrl: string }
+  | { type: 'download-started'; vendor: IconVendor; expectedBytes: number | null }
+  | { type: 'download-progress'; vendor: IconVendor; receivedBytes: number }
+  | { type: 'extracting'; vendor: IconVendor }
+  | { type: 'indexing'; vendor: IconVendor; iconCount: number }
+  | { type: 'done'; vendor: IconVendor; version: string; iconCount: number }
+  | { type: 'error'; vendor: IconVendor; message: string };
+
+export interface InstallOptions {
+  acceptTerms?: boolean;
+  /** Optional URL override for tests; production picks the vendor default. */
+  packUrl?: string;
+}

package/src/icons/installer.ts

@@ -0,0 +1,76 @@
+import { mkdirSync, rmSync } from 'node:fs';
+import { join } from 'node:path';
+import { extractZipToDir } from './extract-zip.ts';
+import { readIndex, upsertPack, writeIndex } from './index-store.ts';
+import type { InstallEvent, InstallOptions } from './installer-types.ts';
+import { withVendorLock } from './lock.ts';
+import type { IconVendor } from './paths.ts';
+import { vendorDescriptor } from './vendors.ts';
+
+export interface InstallerDeps {
+  cacheRoot: string;
+  now: () => number;
+  version: () => string;
+  fetcher: (url: string) => Promise<Buffer>;
+}
+
+export async function* installIconPack(
+  args: { vendor: IconVendor } & InstallOptions,
+  deps: InstallerDeps,
+): AsyncGenerator<InstallEvent> {
+  const desc = vendorDescriptor(args.vendor);
+  if (desc.requiresAcceptance && !args.acceptTerms) {
+    yield { type: 'terms-required', vendor: args.vendor, licenseUrl: desc.licenseUrl };
+    return;
+  }
+
+  const events: InstallEvent[] = [];
+  const lockPath = join(deps.cacheRoot, '.locks', `${args.vendor}.lock`);
+  await withVendorLock(lockPath, async () => {
+    try {
+      events.push({ type: 'download-started', vendor: args.vendor, expectedBytes: null });
+      const buffer = await deps.fetcher(args.packUrl ?? desc.defaultPackUrl);
+
+      events.push({ type: 'extracting', vendor: args.vendor });
+      const version = deps.version();
+      const destDir = join(deps.cacheRoot, args.vendor, version);
+      rmSync(destDir, { recursive: true, force: true });
+      mkdirSync(destDir, { recursive: true });
+      const writtenFilenames = await extractZipToDir(buffer, destDir);
+
+      const icons: Record<string, string> = {};
+      for (const filename of writtenFilenames) {
+        const canonical = desc.canonicalName(filename);
+        if (!canonical) continue;
+        icons[canonical] = `${args.vendor}/${version}/${filename}`;
+      }
+      events.push({ type: 'indexing', vendor: args.vendor, iconCount: Object.keys(icons).length });
+
+      const sizeBytes = Object.values(icons).length;
+      const idx = readIndex(deps.cacheRoot);
+      const next = upsertPack(idx, {
+        vendor: args.vendor,
+        version,
+        installedAt: deps.now(),
+        sizeBytes,
+        icons,
+      });
+      writeIndex(deps.cacheRoot, next);
+
+      events.push({
+        type: 'done',
+        vendor: args.vendor,
+        version,
+        iconCount: Object.keys(icons).length,
+      });
+    } catch (err) {
+      events.push({
+        type: 'error',
+        vendor: args.vendor,
+        message: err instanceof Error ? err.message : String(err),
+      });
+    }
+  });
+
+  for (const ev of events) yield ev;
+}

package/src/icons/jobs.ts

@@ -0,0 +1,78 @@
+import { randomUUID } from 'node:crypto';
+import type { InstallEvent } from './installer-types.ts';
+import type { IconVendor } from './paths.ts';
+
+interface Job {
+  id: string;
+  vendor: IconVendor;
+  events: InstallEvent[];
+  complete: boolean;
+  subscribers: Set<(ev: InstallEvent) => void>;
+  endSubscribers: Set<() => void>;
+}
+
+export interface JobRegistry {
+  create(vendor: IconVendor): string;
+  append(id: string, ev: InstallEvent): void;
+  markComplete(id: string): void;
+  get(id: string): Job | undefined;
+  subscribe(id: string, onEvent: (ev: InstallEvent) => void, onEnd: () => void): () => void;
+  inFlightFor(vendor: IconVendor): string | undefined;
+}
+
+export function createJobRegistry(): JobRegistry {
+  const jobs = new Map<string, Job>();
+  return {
+    create(vendor) {
+      for (const j of jobs.values()) {
+        if (j.vendor === vendor && !j.complete) {
+          throw new Error(`Install for vendor ${vendor} already in flight (job ${j.id})`);
+        }
+      }
+      const id = randomUUID();
+      jobs.set(id, {
+        id,
+        vendor,
+        events: [],
+        complete: false,
+        subscribers: new Set(),
+        endSubscribers: new Set(),
+      });
+      return id;
+    },
+    append(id, ev) {
+      const j = jobs.get(id);
+      if (!j) return;
+      j.events.push(ev);
+      for (const sub of j.subscribers) sub(ev);
+    },
+    markComplete(id) {
+      const j = jobs.get(id);
+      if (!j) return;
+      j.complete = true;
+      for (const onEnd of j.endSubscribers) onEnd();
+    },
+    get: (id) => jobs.get(id),
+    subscribe(id, onEvent, onEnd) {
+      const j = jobs.get(id);
+      if (!j) return () => undefined;
+      for (const ev of j.events) onEvent(ev);
+      if (j.complete) {
+        onEnd();
+        return () => undefined;
+      }
+      j.subscribers.add(onEvent);
+      j.endSubscribers.add(onEnd);
+      return () => {
+        j.subscribers.delete(onEvent);
+        j.endSubscribers.delete(onEnd);
+      };
+    },
+    inFlightFor(vendor) {
+      for (const j of jobs.values()) {
+        if (j.vendor === vendor && !j.complete) return j.id;
+      }
+      return undefined;
+    },
+  };
+}

package/src/icons/list-helper.ts

@@ -0,0 +1,36 @@
+import type { IconIndex } from './index-store.ts';
+import type { IconVendor } from './paths.ts';
+
+const ALL: IconVendor[] = ['aws', 'azure'];
+
+export type PackSummary =
+  | {
+      vendor: IconVendor;
+      installed: true;
+      version: string;
+      iconCount: number;
+      sizeBytes: number;
+      /**
+       * Canonical icon names (kebab-case) installed in this pack, sorted
+       * alphabetically. Mirrors the canvas-side PackSummary so apps/web can
+       * pass the wire JSON straight through to the picker's vendor grids.
+       */
+      iconNames: string[];
+    }
+  | { vendor: IconVendor; installed: false };
+
+export function summarizePacks(idx: IconIndex): PackSummary[] {
+  return ALL.map((vendor) => {
+    const p = idx.packs[vendor];
+    if (!p) return { vendor, installed: false };
+    const names = Object.keys(p.icons).sort();
+    return {
+      vendor,
+      installed: true,
+      version: p.version,
+      iconCount: names.length,
+      sizeBytes: p.sizeBytes,
+      iconNames: names,
+    };
+  });
+}

package/src/icons/lock.ts

@@ -0,0 +1,13 @@
+const queues = new Map<string, Promise<unknown>>();
+
+export async function withVendorLock<T>(lockPath: string, fn: () => Promise<T>): Promise<T> {
+  const prev = queues.get(lockPath) ?? Promise.resolve();
+  const run = prev.then(fn, fn);
+  const tracked = run.catch(() => undefined);
+  queues.set(lockPath, tracked);
+  try {
+    return await run;
+  } finally {
+    if (queues.get(lockPath) === tracked) queues.delete(lockPath);
+  }
+}

package/src/icons/normalize-aws.ts

@@ -0,0 +1,19 @@
+const STRIP_PREFIXES = ['Arch_AWS-', 'Arch_Amazon-', 'Arch-Category_', 'Arch_'];
+const SIZE_SUFFIX = /_(?:16|32|48|64)$/;
+
+export function canonicalAwsName(filename: string): string | null {
+  if (!filename.toLowerCase().endsWith('.svg')) return null;
+  let base = filename.slice(0, -'.svg'.length);
+  for (const prefix of STRIP_PREFIXES) {
+    if (base.startsWith(prefix)) {
+      base = base.slice(prefix.length);
+      break;
+    }
+  }
+  base = base.replace(SIZE_SUFFIX, '');
+  const kebab = base
+    .toLowerCase()
+    .replace(/[^a-z0-9]+/g, '-')
+    .replace(/^-+|-+$/g, '');
+  return kebab.length > 0 ? kebab : null;
+}

package/src/icons/normalize-azure.ts

@@ -0,0 +1,11 @@
+export function canonicalAzureName(filename: string): string | null {
+  if (!filename.toLowerCase().endsWith('.svg')) return null;
+  let base = filename.slice(0, -'.svg'.length);
+  base = base.replace(/^\d+-icon-service-/i, '');
+  base = base.replace(/^icon-service-/i, '');
+  const kebab = base
+    .toLowerCase()
+    .replace(/[^a-z0-9]+/g, '-')
+    .replace(/^-+|-+$/g, '');
+  return kebab.length > 0 ? kebab : null;
+}

package/src/icons/paths.ts

@@ -0,0 +1,14 @@
+import { join } from 'node:path';
+import { seeflowHome } from '../paths.ts';
+
+export type IconVendor = 'aws' | 'azure';
+
+export const iconCacheRoot = (): string => join(seeflowHome(), 'icons');
+
+export const iconVendorRoot = (vendor: IconVendor, version: string): string =>
+  join(iconCacheRoot(), vendor, version);
+
+export const iconLockPath = (vendor: IconVendor): string =>
+  join(iconCacheRoot(), '.locks', `${vendor}.lock`);
+
+export const iconIndexPath = (): string => join(iconCacheRoot(), 'index.json');

package/src/icons/remove.ts

@@ -0,0 +1,16 @@
+import { mkdirSync, rmSync } from 'node:fs';
+import { join } from 'node:path';
+import { readIndex, removePack, writeIndex } from './index-store.ts';
+import type { IconVendor } from './paths.ts';
+
+export interface RemoveIconPackDeps {
+  cacheRoot: string;
+}
+
+export function removeIconPack(vendor: IconVendor, deps: RemoveIconPackDeps): void {
+  rmSync(join(deps.cacheRoot, vendor), { recursive: true, force: true });
+  const idx = readIndex(deps.cacheRoot);
+  if (!idx.packs[vendor]) return;
+  mkdirSync(deps.cacheRoot, { recursive: true });
+  writeIndex(deps.cacheRoot, removePack(idx, vendor));
+}