@ai-ecoverse/slicc-man 0.0.0 → 1.1.0

package/lib/render.mjs

@@ -0,0 +1,214 @@
+import { BOLD, inline, parse, plain, UNDERLINE } from './markdown.mjs';
+
+const BODY = 7;
+const SUB = 3;
+
+function paint(cells, styled) {
+  let out = '';
+  let current = 0;
+  for (const cell of cells) {
+    const style = styled ? cell.style : 0;
+    if (style !== current) {
+      if (current !== 0) out += '\x1b[0m';
+      if (style & BOLD) out += '\x1b[1m';
+      if (style & UNDERLINE) out += '\x1b[4m';
+      current = style;
+    }
+    out += cell.ch;
+  }
+  if (current !== 0) out += '\x1b[0m';
+  return out;
+}
+
+function words(runs) {
+  const lines = [[]];
+  let word = [];
+  const end = () => {
+    if (word.length > 0) lines.at(-1).push(word);
+    word = [];
+  };
+  for (const run of runs) {
+    if (run.br) {
+      end();
+      lines.push([]);
+      continue;
+    }
+    for (const ch of run.text) {
+      if (/\s/u.test(ch) && ch !== '\u00a0') end();
+      else word.push({ ch, style: run.style });
+    }
+  }
+  end();
+  return lines;
+}
+
+function wrap(runs, indent, width, styled, lead = '') {
+  const hang = ' '.repeat(indent + lead.length);
+  const room = Math.max(width - indent - lead.length, 1);
+  const out = [];
+  let first = true;
+  for (const line of words(runs)) {
+    let cur = [];
+    let size = 0;
+    const emit = () => {
+      const text = cur.map((w) => paint(w, styled)).join(' ');
+      out.push(`${first ? ' '.repeat(indent) + lead : hang}${text}`.trimEnd());
+      first = false;
+      cur = [];
+      size = 0;
+    };
+    for (const w of line) {
+      if (cur.length > 0 && size + 1 + w.length > room) emit();
+      size += (cur.length > 0 ? 1 : 0) + w.length;
+      cur.push(w);
+    }
+    if (cur.length > 0 || out.length === 0 || line.length === 0) emit();
+  }
+  return out;
+}
+
+function upper(runs) {
+  return runs.map((run) => ({ ...run, text: run.text.toUpperCase() }));
+}
+
+function table(block, indent, styled) {
+  const rows = block.rows.map((row) => {
+    const out = [];
+    for (let c = 0; c < block.columns; c++) {
+      const runs = inline(row[c] ?? '');
+      const cells = [];
+      for (const line of words(runs)) {
+        for (const w of line) {
+          if (cells.length > 0) cells.push({ ch: ' ', style: 0 });
+          cells.push(...w);
+        }
+      }
+      out.push(cells);
+    }
+    return out;
+  });
+  const widths = [];
+  for (let c = 0; c < block.columns; c++) {
+    widths.push(Math.max(3, ...rows.map((row) => row[c].length)));
+  }
+  const line = (row) =>
+    `${' '.repeat(indent)}| ${row
+      .map((cells, c) => paint(cells, styled) + ' '.repeat(widths[c] - cells.length))
+      .join(' | ')} |`;
+  const rule = `${' '.repeat(indent)}| ${widths.map((w) => '-'.repeat(w)).join(' | ')} |`;
+  return [line(rows[0]), rule, ...rows.slice(1).map(line)];
+}
+
+function emit(block, indent, ctx, out) {
+  const { width, styled } = ctx;
+  switch (block.type) {
+    case 'paragraph':
+      out.push(...wrap(inline(block.text), indent, width, styled));
+      break;
+    case 'code':
+      out.push(...block.lines.map((line) => `${' '.repeat(indent)}${line}`.trimEnd()));
+      break;
+    case 'grid':
+      out.push(...block.lines.map((line) => `${' '.repeat(indent)}${line}`));
+      break;
+    case 'table':
+      out.push(...table(block, indent, styled));
+      break;
+    case 'quote':
+      sequence(block.blocks, indent + 4, ctx, out);
+      break;
+    case 'list':
+      for (const [index, entry] of block.items.entries()) {
+        if (index > 0 && block.loose) out.push('');
+        const lead = `${entry.marker} `;
+        const [head, ...rest] = entry.blocks;
+        if (head?.type === 'paragraph') {
+          out.push(...wrap(inline(head.text), indent, width, styled, lead));
+        } else {
+          out.push(`${' '.repeat(indent)}${entry.marker}`);
+          if (head) rest.unshift(head);
+        }
+        for (const inner of rest) {
+          if (block.loose || inner.type !== 'list') out.push('');
+          emit(inner, indent + lead.length, ctx, out);
+        }
+      }
+      break;
+    case 'heading': {
+      const runs = inline(block.text);
+      if (block.level <= 2) out.push(paint(cellsOf(upper(runs)), styled));
+      else out.push(' '.repeat(SUB) + paint(cellsOf(runs), styled));
+      break;
+    }
+  }
+}
+
+function cellsOf(runs) {
+  const cells = [];
+  for (const line of words(runs)) {
+    for (const w of line) {
+      if (cells.length > 0) cells.push({ ch: ' ', style: BOLD });
+      cells.push(...w.map((cell) => ({ ...cell, style: cell.style | BOLD })));
+    }
+  }
+  return cells;
+}
+
+function sequence(list, indent, ctx, out) {
+  let after = true;
+  for (const block of list) {
+    if (block.type === 'rule') continue;
+    if (block.type === 'heading') {
+      if (out.length > 0) out.push('');
+      emit(block, indent, ctx, out);
+      after = true;
+      continue;
+    }
+    if (!after) out.push('');
+    emit(block, indent, ctx, out);
+    after = false;
+  }
+}
+
+function metadata(block) {
+  if (block?.type === 'grid') return block.first.join(' ').trim() === 'Metadata';
+  if (block?.type === 'table') return block.rows[0].join(' ').trim() === 'Metadata';
+  return false;
+}
+
+function header(topic, width) {
+  const side = `${topic.toUpperCase()}(1)`;
+  const middle = 'SLICC Manual';
+  const free = width - side.length * 2 - middle.length;
+  if (free < 2) return `${side} ${middle} ${side}`;
+  const left = Math.max(1, Math.floor((width - middle.length) / 2) - side.length);
+  const right = free - left;
+  if (right < 1) return `${side}${' '.repeat(free - 1)}${middle} ${side}`;
+  return `${side}${' '.repeat(left)}${middle}${' '.repeat(right)}${side}`;
+}
+
+export function render(markdown, { topic, width = 80, styled = false }) {
+  const list = parse(markdown);
+  while (list.length > 0 && list.at(-1).type === 'rule') list.pop();
+  if (metadata(list.at(-1))) list.pop();
+  const ctx = { width, styled };
+  const out = [header(topic, width), ''];
+  out.push(paint(cellsOf([{ text: 'NAME', style: 0 }]), styled));
+  const name = [{ text: topic, style: 0 }];
+  if (list[0]?.type === 'paragraph') {
+    name.push({ text: ' - ', style: 0 }, ...inline(list.shift().text));
+  }
+  out.push(...wrap(name, BODY, width, styled));
+  const body = [];
+  sequence(list, BODY, ctx, body);
+  if (body.length > 0) {
+    if (body[0] !== '') out.push('');
+    out.push(...body);
+  }
+  return `${out.join('\n')}\n`;
+}
+
+export function describe(markdown) {
+  const first = parse(markdown)[0];
+  return first?.type === 'paragraph' ? plain(inline(first.text)) : '';
+}

package/lib/search.mjs

@@ -0,0 +1,83 @@
+const special = /[.*+?^${}()|[\]\\/-]/g;
+
+function bracket(pattern, i) {
+  let j = i + 1;
+  if (pattern[j] === '!' || pattern[j] === '^') j++;
+  if (pattern[j] === ']') j++;
+  j = pattern.indexOf(']', j);
+  if (j === -1) return { text: '\\[', next: i + 1 };
+  let body = pattern.slice(i + 1, j);
+  const not = body[0] === '!' || body[0] === '^';
+  if (not) body = body.slice(1);
+  return { text: `[${not ? '^' : ''}${body.replace(/[\\\]^]/g, '\\$&')}]`, next: j + 1 };
+}
+
+function glob(pattern) {
+  let out = '';
+  let i = 0;
+  while (i < pattern.length) {
+    const ch = pattern[i];
+    if (ch === '[') {
+      const found = bracket(pattern, i);
+      out += found.text;
+      i = found.next;
+      continue;
+    }
+    if (ch === '*') out += '.*';
+    else if (ch === '?') out += '.';
+    else if (ch === '\\' && i + 1 < pattern.length) out += pattern[++i].replace(special, '\\$&');
+    else out += ch.replace(special, '\\$&');
+    i++;
+  }
+  return new RegExp(`^(?:${out})$`, 'i');
+}
+
+export function words(text) {
+  return text.split(/[^\p{L}\p{N}_]+/u).filter(Boolean);
+}
+
+export function matcher(kind, mode, keyword) {
+  if (mode === 'regex') {
+    const re = new RegExp(keyword, 'i');
+    return kind === 'whatis'
+      ? (entry) => re.test(entry.topic)
+      : (entry) => re.test(entry.topic) || re.test(entry.description);
+  }
+  if (mode === 'wildcard') {
+    const re = glob(keyword);
+    return kind === 'whatis'
+      ? (entry) => re.test(entry.topic)
+      : (entry) => re.test(entry.topic) || words(entry.description).some((w) => re.test(w));
+  }
+  const want = keyword.toLowerCase();
+  return kind === 'whatis'
+    ? (entry) => entry.topic.toLowerCase() === want
+    : (entry) =>
+        entry.topic.toLowerCase() === want ||
+        words(entry.description).some((w) => w.toLowerCase() === want);
+}
+
+export function line(entry) {
+  return `${`${entry.topic} (1)`.padEnd(20)} - ${entry.description}`.trimEnd();
+}
+
+export function cut(text, width) {
+  const chars = Array.from(text);
+  if (chars.length <= width) return text;
+  return `${chars.slice(0, Math.max(width - 3, 0)).join('')}...`;
+}
+
+export function search(entries, matchers, all) {
+  const hits = matchers.map(() => false);
+  const found = [];
+  for (const entry of entries) {
+    const each = matchers.map((match) => match(entry));
+    const ok = all ? each.every(Boolean) : each.some(Boolean);
+    if (!ok) continue;
+    found.push(entry);
+    each.forEach((hit, n) => {
+      if (hit) hits[n] = true;
+    });
+  }
+  return { found, hits };
+}

package/lib/site.mjs

@@ -0,0 +1,205 @@
+import { describe } from './render.mjs';
+
+export const DEFAULT_BASE = 'https://www.sliccy.com';
+export const FRESH = 4 * 60 * 60;
+const TIMEOUT = 30000;
+const PARALLEL = 8;
+
+export function valid(topic) {
+  return /^[A-Za-z0-9_+-][A-Za-z0-9._+-]*$/.test(topic);
+}
+
+export function base(env) {
+  return (env.SLICC_MAN_URL || DEFAULT_BASE).replace(/\/+$/, '');
+}
+
+export function folder(url) {
+  const parsed = new URL(url);
+  const host = parsed.hostname.replace(/[^A-Za-z0-9.-]/g, '_');
+  return parsed.port ? `${host}-${parsed.port}` : host;
+}
+
+export function key(topic) {
+  return `page-${topic.replace(/[A-Z_]/g, (ch) => (ch === '_' ? '__' : `_${ch.toLowerCase()}`))}`;
+}
+
+function age(headers) {
+  const match = /(?:^|[,\s])max-age\s*=\s*"?(\d+)"?/i.exec(headers.get('cache-control') ?? '');
+  return match ? Number(match[1]) : undefined;
+}
+
+function reason(error) {
+  if (error?.name === 'TimeoutError') return 'timed out';
+  const cause = error?.cause;
+  return cause?.message || cause?.code || error?.message || String(error);
+}
+
+function stamp(time) {
+  return new Date(time).toISOString().replace(/\.\d+Z$/, 'Z');
+}
+
+export function topics(xml) {
+  const seen = new Set();
+  const out = [];
+  for (const match of String(xml).matchAll(/<loc>\s*([^<]*?)\s*<\/loc>/g)) {
+    const text = match[1]
+      .replace(/&lt;/g, '<')
+      .replace(/&gt;/g, '>')
+      .replace(/&quot;/g, '"')
+      .replace(/&apos;/g, "'")
+      .replace(/&amp;/g, '&');
+    let path;
+    try {
+      path = new URL(text, 'https://localhost').pathname;
+    } catch {
+      continue;
+    }
+    const found = /^\/man\/([^/]+)$/.exec(path);
+    if (!found) continue;
+    let topic;
+    try {
+      topic = decodeURIComponent(found[1]);
+    } catch {
+      continue;
+    }
+    if (!valid(topic) || seen.has(topic)) continue;
+    seen.add(topic);
+    out.push(topic);
+  }
+  return out;
+}
+
+export function site({ url, store, fetch, now = Date.now, name }) {
+  const root = url.replace(/\/+$/, '');
+
+  async function load(id) {
+    try {
+      const text = await store.read(id);
+      if (text === undefined) return undefined;
+      const entry = JSON.parse(text);
+      return entry && typeof entry === 'object' ? entry : undefined;
+    } catch {
+      return undefined;
+    }
+  }
+
+  async function save(id, entry) {
+    try {
+      await store.write(id, JSON.stringify(entry));
+    } catch {}
+  }
+
+  async function drop(id) {
+    try {
+      await store.remove(id);
+    } catch {}
+  }
+
+  const fresh = (entry) =>
+    typeof entry?.fetched === 'number' && now() - entry.fetched < (entry.maxAge ?? FRESH) * 1000;
+
+  async function request(address, id, entry) {
+    const headers = {};
+    if (entry?.lastModified) headers['if-modified-since'] = entry.lastModified;
+    const response = await fetch(address, { headers, signal: AbortSignal.timeout(TIMEOUT) });
+    if (response.status === 304 && typeof entry?.body === 'string') {
+      await response.body?.cancel();
+      await save(id, { ...entry, fetched: now(), maxAge: age(response.headers) ?? entry.maxAge });
+      return { status: 'ok', body: entry.body };
+    }
+    if (response.status === 200) {
+      const body = await response.text();
+      const lastModified = response.headers.get('last-modified') ?? undefined;
+      const maxAge = age(response.headers) ?? FRESH;
+      await save(id, { fetched: now(), maxAge, lastModified, body });
+      return { status: 'ok', body };
+    }
+    await response.body?.cancel();
+    if (response.status === 404 || response.status === 410) {
+      if (entry) await drop(id);
+      return { status: 'missing' };
+    }
+    return { why: `HTTP ${response.status}` };
+  }
+
+  async function get(address, id, { offline = false } = {}) {
+    const entry = await load(id);
+    const kept = typeof entry?.body === 'string' && typeof entry.fetched === 'number';
+    if (kept && (fresh(entry) || offline)) return { status: 'ok', body: entry.body };
+    if (offline) return { status: 'error', message: `${name}: ${address}: offline` };
+    let found;
+    try {
+      found = await request(address, id, entry);
+    } catch (error) {
+      found = { why: reason(error) };
+    }
+    if (found.status) return found;
+    if (!kept)
+      return { status: 'error', message: `${name}: ${address}: ${found.why}`, offline: true };
+    return {
+      status: 'ok',
+      body: entry.body,
+      stale: true,
+      note: `${name}: ${address}: ${found.why}; showing the copy cached ${stamp(entry.fetched)}`,
+    };
+  }
+
+  const address = (topic) => `${root}/man/${topic}.md`;
+
+  async function page(topic, options) {
+    if (!valid(topic)) return { status: 'missing' };
+    return get(address(topic), key(topic), options);
+  }
+
+  async function build(list, offline) {
+    const entries = new Array(list.length);
+    let next = 0;
+    let down = offline;
+    async function worker() {
+      while (next < list.length) {
+        const at = next++;
+        const topic = list[at];
+        const found = await page(topic, { offline: down });
+        if (found.stale || found.offline) down = true;
+        entries[at] = { topic, description: found.status === 'ok' ? describe(found.body) : '' };
+      }
+    }
+    await Promise.all(Array.from({ length: Math.min(PARALLEL, list.length) }, worker));
+    return { entries, down };
+  }
+
+  async function index() {
+    const cached = await load('index');
+    const usable = Array.isArray(cached?.entries);
+    if (usable && fresh(cached)) return { entries: cached.entries };
+    const map = `${root}/sitemap.xml`;
+    const found = await get(map, 'sitemap');
+    if (found.status !== 'ok') {
+      const message = found.message ?? `${name}: ${map}: HTTP 404`;
+      if (usable && typeof cached.fetched === 'number') {
+        const why = message.slice(`${name}: ${map}: `.length);
+        return {
+          entries: cached.entries,
+          note: `${name}: ${map}: ${why}; showing the copy cached ${stamp(cached.fetched)}`,
+        };
+      }
+      return { error: message };
+    }
+    if (found.stale) {
+      if (usable) return { entries: cached.entries, note: found.note };
+      return { entries: (await build(topics(found.body), true)).entries, note: found.note };
+    }
+    const sitemap = await load('sitemap');
+    const built = await build(topics(found.body), false);
+    if (!built.down) {
+      await save('index', {
+        fetched: now(),
+        maxAge: sitemap?.maxAge ?? FRESH,
+        entries: built.entries,
+      });
+    }
+    return { entries: built.entries };
+  }
+
+  return { address, page, index };
+}

package/package.json

@@ -1,10 +1,56 @@
 {
   "name": "@ai-ecoverse/slicc-man",
-  "version": "0.0.0",
+  "version": "1.1.0",
   "description": "man, apropos and whatis for SLICC, and scman for SLICC, macOS, Linux and Windows, reading pages from www.sliccy.com/man",
+  "type": "module",
   "license": "Apache-2.0",
   "repository": {
     "type": "git",
     "url": "git+https://github.com/ai-ecoverse/slicc-man.git"
+  },
+  "engines": {
+    "node": ">=24"
+  },
+  "bin": {
+    "scman": "host/scman.mjs"
+  },
+  "fledgling": {
+    "provider": "github",
+    "workflow": "ci.yml",
+    "repo": "ai-ecoverse/slicc-man",
+    "publish": true
+  },
+  "publishConfig": {
+    "access": "public",
+    "provenance": true
+  },
+  "files": [
+    "host",
+    "lib"
+  ],
+  "slicc": {
+    "unitSources": [
+      "host",
+      "lib",
+      "scripts"
+    ]
+  },
+  "scripts": {
+    "prepare": "slicc-install-hooks",
+    "lint": "slicc-lint && shellcheck -s sh install.sh && dash -n install.sh",
+    "format": "biome check --write .",
+    "test": "node --test --test-timeout=120000 test/integration/*.test.mjs",
+    "test:unit": "node scripts/cover.mjs",
+    "build": "npm pack --dry-run",
+    "trust": "npx fledgling sync -y --skip-publish",
+    "trust:dry": "npx fledgling sync --dry-run -y --skip-publish",
+    "release": "semantic-release"
+  },
+  "devDependencies": {
+    "@ai-ecoverse/slicc-shared-web": "2.11.3",
+    "@biomejs/biome": "2.5.15",
+    "@semantic-release/exec": "7.1.0",
+    "lefthook": "2.2.2",
+    "semantic-release": "25.0.9"
   }
 }