@webship.co/workspace 0.0.0-stage → 1.0.0-beta1

package/rag.js

@@ -0,0 +1,169 @@
+/**
+ * The RAG indexes: which Milvus server holds a project's, and how the dashboard reaches it.
+ *
+ * Not the published host port — this process runs inside ddev-workspace-web, where 127.0.0.1 is
+ * that container. DDEV puts every project on the shared `ddev_default` network, so the answer here
+ * is the Milvus container's own name.
+ */
+
+const fs = require('fs');
+const http = require('http');
+const path = require('path');
+
+const { ROOT, CONFIG_DIR } = require('./workspaces');
+
+/* ---------------- RAG indexes (Milvus) ---------------- */
+//
+// The counterpart of the graph helpers above, and deliberately the layer graphify does not cover:
+// an index holds the TEXT of a project (Markdown, Twig, *.yml, hooks) as rows in Milvus, one
+// collection per project — `ws_<workspace>_<project>`, the same name cmd-tools-ragify.sh writes.
+//
+// The database is not a file, so unlike a graph there is nothing on disk to stat: whether a project
+// has an index is a question for Milvus, asked over its REST API and memoised, because
+// projectRowsHtml re-renders on every job poll.
+
+// Which Milvus instance, from the rag workspace's own settings. Read line by line rather than with
+// a YAML parser, for the same reason the settings editor does: this file is written by hand and
+// keeps its comments.
+function milvusProjectName() {
+  try {
+    const text = fs.readFileSync(path.join(CONFIG_DIR, 'workspace.rag.settings.yml'), 'utf8');
+    const inMilvus = text.split('\n');
+    let block = false;
+    for (const line of inMilvus) {
+      if (/^milvus:\s*$/.test(line)) { block = true; continue; }
+      if (block && /^\S/.test(line)) break;
+      const m = block && line.match(/^\s{2}project:\s*"?([a-zA-Z0-9_-]+)"?\s*$/);
+      if (m) return m[1];
+    }
+  } catch (_) { /* not configured */ }
+  return 'milvus';
+}
+
+// Where the dashboard reaches Milvus. NOT the published host port: this process runs inside
+// ddev-workspace-web, where 127.0.0.1 is this container and the host's firewall drops
+// container-to-host traffic anyway. DDEV puts every project on the shared `ddev_default` network,
+// so the Milvus container answers on its own name — same port, gRPC and REST multiplexed.
+function milvusRestBase(instance) {
+  return `http://ddev-${instance || milvusProjectName()}-milvus-standalone:19530`;
+}
+
+// Every RAG server built in the rag workspace: a directory carrying the compose file this tooling
+// writes. One shared instance is the normal case; a project can be bound to its own.
+function ragInstances() {
+  const base = path.join(ROOT, 'rag');
+  let entries = [];
+  try { entries = fs.readdirSync(base, { withFileTypes: true }); } catch (_) { return []; }
+  return entries
+    .filter((e) => e.isDirectory() && fs.existsSync(path.join(base, e.name, '.ddev', 'docker-compose.milvus.yaml')))
+    .map((e) => e.name)
+    .sort();
+}
+
+// Which server a project's index lives in. rag/bindings.yml holds one line per BOUND project —
+// `<workspace>/<project>: <instance>` — and nothing for the rest, which use the default. The file
+// is machine-local state and lives in the rag workspace rather than in the project, for the same
+// reason a graph does: a repository we ship must not gain a file because somebody indexed it.
+let _bindings = null;
+let _bindingsAt = 0;
+function ragBindings() {
+  const now = Date.now();
+  if (_bindings && now - _bindingsAt < 2000) return _bindings;
+  const map = new Map();
+  try {
+    const text = fs.readFileSync(path.join(ROOT, 'rag', 'bindings.yml'), 'utf8');
+    for (const line of text.split('\n')) {
+      const m = line.match(/^([a-z0-9_-]+\/[A-Za-z0-9_.-]+):\s*([A-Za-z0-9_-]+)\s*$/);
+      if (m) map.set(m[1], m[2]);
+    }
+  } catch (_) { /* nothing bound */ }
+  _bindings = map;
+  _bindingsAt = now;
+  return map;
+}
+
+function ragInstanceFor(wsKey, project) {
+  return ragBindings().get(`${wsKey}/${project}`) || milvusProjectName();
+}
+
+function ragWriteBinding(wsKey, project, instance) {
+  const file = path.join(ROOT, 'rag', 'bindings.yml');
+  let lines = [];
+  try {
+    lines = fs.readFileSync(file, 'utf8').split('\n');
+  } catch (_) {
+    lines = ['# Which Milvus instance holds each project\'s RAG index.',
+      '#',
+      '# Written by cmd-tools-ragify.sh --milvus <instance> and by the dashboard\'s RAG menu. A project',
+      '# that is not listed here uses milvus.project from core/config/workspace.rag.settings.yml.',
+      '#',
+      '# Machine-local state, like the graphs store: it describes what is built on THIS machine.', ''];
+  }
+  const key = `${wsKey}/${project}`;
+  const kept = lines.filter((l) => !l.startsWith(`${key}:`));
+  if (instance) kept.push(`${key}: ${instance}`);
+  fs.writeFileSync(file, `${kept.filter((l, i, a) => !(l === '' && a[i + 1] === '')).join('\n').replace(/\n+$/, '')}\n`, 'utf8');
+  _bindingsAt = 0;
+}
+
+// Milvus identifiers allow letters, digits and underscore only — the same folding
+// rag_collection_name() does in bash and collection_ok() does in the Python worker.
+function ragCollectionName(wsKey, project) {
+  return `ws_${wsKey}_${project}`.replace(/[^A-Za-z0-9_]/g, '_');
+}
+
+function milvusPost(pathname, body, timeoutMs = 4000, instance = null) {
+  return new Promise((resolve) => {
+    const payload = JSON.stringify(body || {});
+    const url = new URL(milvusRestBase(instance) + pathname);
+    const request = http.request({
+      hostname: url.hostname,
+      port: url.port,
+      path: url.pathname,
+      method: 'POST',
+      timeout: timeoutMs,
+      headers: { 'Content-Type': 'application/json', 'Content-Length': Buffer.byteLength(payload) },
+    }, (response) => {
+      let data = '';
+      response.on('data', (chunk) => { data += chunk; });
+      response.on('end', () => {
+        try { resolve(JSON.parse(data)); } catch (_) { resolve(null); }
+      });
+    });
+    // A stopped instance must not stall a page render: every caller treats null as "no Milvus".
+    request.on('error', () => resolve(null));
+    request.on('timeout', () => { request.destroy(); resolve(null); });
+    request.end(payload);
+  });
+}
+
+// Every collection in one server, memoised for 5s and keyed by server. One request per server per
+// render rather than one per project row, and a stopped Milvus costs one connection refusal, not
+// one per row. Keyed, because a workspace can mix projects bound to different servers.
+const _ragCache = new Map();      // instance -> { list, up, at }
+const _ragInFlight = new Map();
+async function ragCollections(instance) {
+  const key = instance || milvusProjectName();
+  const hit = _ragCache.get(key);
+  if (hit && Date.now() - hit.at < 5000) return hit;
+  if (!_ragInFlight.has(key)) {
+    _ragInFlight.set(key, milvusPost('/v2/vectordb/collections/list', {}, 4000, key).then((body) => {
+      const up = !!body && body.code === 0;
+      const entry = { list: up && Array.isArray(body.data) ? body.data : [], up, at: Date.now() };
+      _ragCache.set(key, entry);
+      _ragInFlight.delete(key);
+      return entry;
+    }));
+  }
+  return _ragInFlight.get(key);
+}
+
+function ragInvalidate() {
+  _ragCache.clear();
+  _bindingsAt = 0;
+}
+
+module.exports = {
+  milvusProjectName, milvusRestBase, ragInstances, ragBindings, ragInstanceFor, ragWriteBinding,
+  ragCollectionName, milvusPost, ragCollections, ragInvalidate,
+};

package/reader.js

@@ -0,0 +1,132 @@
+/**
+ * The avatar's browser window: a web page, read through this server.
+ *
+ * Most pages a team needs (drupal.org, GitHub, MDN) refuse to be shown inside another page, so the
+ * window cannot simply frame them. GET /fragments/reader?u=<https url> fetches the page here and
+ * serves its main content back, cleaned: no scripts, frames, forms, styles from elsewhere or event
+ * handlers, every link pointing back into the reader, and a Content-Security-Policy that allows none
+ * of it to run anyway.
+ *
+ * The fetch happens inside the dashboard's container, which can reach the Docker network and every
+ * DDEV project, so an address is only fetched when it is https, a name rather than an IP, and every
+ * address it resolves to is public; each redirect is checked the same way. A page that answers a
+ * program with a bot check (drupal.org does) is said so plainly, with the link to open in a tab:
+ * the reader never tries to get past one.
+ */
+
+const dns = require('dns').promises;
+const net = require('net');
+const { esc } = require('./html');
+
+const MAX_BYTES = 1500000;
+
+function privateIp(ip) {
+  if (net.isIPv4(ip)) {
+    const [a, b] = ip.split('.').map(Number);
+    return a === 10 || a === 127 || a === 0 || (a === 169 && b === 254) || (a === 172 && b >= 16 && b <= 31)
+      || (a === 192 && b === 168) || (a === 100 && b >= 64 && b <= 127) || a >= 224;
+  }
+  const v = ip.toLowerCase();
+  return v === '::1' || v === '::' || v.startsWith('fc') || v.startsWith('fd') || v.startsWith('fe80') || v.startsWith('::ffff:');
+}
+
+/** Whether a URL may be fetched: https, a public name, every address it resolves to public. */
+async function allowed(u, lookup = dns.lookup) {
+  if (u.protocol !== 'https:') return 'only https pages can be opened';
+  if (u.port && u.port !== '443') return 'only the standard https port';
+  const host = u.hostname.toLowerCase();
+  if (net.isIP(host) || !/^[a-z0-9.-]+\.[a-z]{2,}$/.test(host)) return 'only website names, not addresses';
+  if (/(^|\.)(ddev\.site|localhost|local|internal|lan)$/.test(host)) return 'local sites open in their own tab';
+  try {
+    const addrs = await lookup(host, { all: true });
+    if (!addrs.length || addrs.some((a) => privateIp(a.address))) return 'that address is not a public website';
+  } catch (_) { return 'that site could not be found'; }
+  return '';
+}
+
+const isChallenge = (html) => /<title[^>]*>\s*(Client Challenge|Just a moment|Attention Required|Verifying you are human)/i
+  .test(String(html || '').slice(0, 20000));
+
+const readerLink = (abs) => `/fragments/reader?u=${encodeURIComponent(abs)}`;
+
+/** The page's main content, cleaned, with its links pointing back into the reader. */
+function cleanHtml(html, base) {
+  let b = (html.match(/<main\b[^>]*>([\s\S]*?)<\/main>/i) || html.match(/<article\b[^>]*>([\s\S]*?)<\/article>/i) || [])[1];
+  if (!b) {
+    b = (html.match(/<body[^>]*>([\s\S]*)<\/body>/i) || [, html])[1];
+    b = b.replace(/<(nav|header|footer|aside)\b[\s\S]*?<\/\1\s*>/gi, '');
+  }
+  b = b.replace(/<!--[\s\S]*?-->/g, '')
+    .replace(/<(script|style|noscript|iframe|object|template|svg|math|canvas|video|audio|dialog)\b[\s\S]*?<\/\1\s*>/gi, '')
+    .replace(/<\/?(script|style|noscript|iframe|object|embed|form|input|button|select|textarea|label|link|meta|base|template|source|track|frame|frameset)\b[^>]*>/gi, '')
+    .replace(/\s(on\w+|style|srcset|sizes|formaction|ping|xmlns(?::\w+)?)\s*=\s*("[^"]*"|'[^']*'|[^\s>]+)/gi, '')
+    .replace(/javascript:/gi, '');
+  b = b.replace(/<a\b[^>]*?href\s*=\s*("([^"]*)"|'([^']*)')[^>]*>/gi, (m, q, h1, h2) => {
+    const href = (h1 ?? h2 ?? '').replace(/&amp;/g, '&');
+    if (href.startsWith('#')) return `<a href="${esc(href)}">`;
+    let abs;
+    try { abs = new URL(href, base); } catch (_) { return '<a>'; }
+    if (abs.protocol !== 'https:' && abs.protocol !== 'http:') return '<a>';
+    abs.protocol = 'https:';
+    return `<a href="${esc(readerLink(abs.href))}" title="${esc(abs.href)}">`;
+  });
+  b = b.replace(/<img\b([^>]*?)src\s*=\s*("([^"]*)"|'([^']*)')([^>]*)>/gi, (m, pre, q, s1, s2, post) => {
+    let abs;
+    try { abs = new URL((s1 ?? s2 ?? '').replace(/&amp;/g, '&'), base); } catch (_) { return ''; }
+    if (abs.protocol !== 'https:') return '';
+    const alt = (`${pre} ${post}`.match(/\salt\s*=\s*"([^"]*)"/i) || [])[1] || '';
+    return `<img src="${esc(abs.href)}" alt="${esc(alt)}" loading="lazy">`;
+  });
+  return b;
+}
+
+function page(title, body) {
+  return `<!doctype html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1">
+<title>${esc(title)}</title><style>
+body { margin: 0; padding: .9rem 1.1rem 2.5rem; font: 15px/1.65 system-ui, sans-serif; color: #1d1d1f; background: #fff; }
+h1, h2, h3 { color: #2b447c; line-height: 1.3; } h1 { font-size: 1.35rem; }
+a { color: #0063c0; text-decoration: none; } a:hover { text-decoration: underline; }
+img { max-width: 100%; height: auto; } table { border-collapse: collapse; display: block; overflow-x: auto; font-size: .9rem; }
+td, th { border: 1px solid #ddd; padding: .25rem .4rem; }
+pre { white-space: pre-wrap; overflow-wrap: anywhere; background: #f5f6f8; padding: .6rem .8rem; border-radius: .4rem; font-size: .85rem; }
+code { background: #f5f6f8; padding: 0 .2rem; border-radius: .2rem; } pre code { background: none; padding: 0; }
+.hint { color: #666; font-size: .85rem; }
+</style></head><body>${body}</body></html>`;
+}
+
+/** GET /fragments/reader?u=… */
+async function serveReader(res, raw, { fetchImpl = fetch, lookup = dns.lookup } = {}) {
+  const out = (status, html) => {
+    res.writeHead(status, {
+      'Content-Type': 'text/html; charset=utf-8',
+      'Content-Security-Policy': "default-src 'none'; img-src https: data:; style-src 'unsafe-inline'; base-uri 'none'; form-action 'none'",
+      'Cache-Control': 'no-store',
+      'X-Content-Type-Options': 'nosniff',
+    });
+    res.end(html);
+  };
+  let u;
+  try { u = new URL(String(raw || '')); } catch (_) { return out(400, page('Not a page', '<p class="hint">That is not a link.</p>')); }
+  const refuse = (why, status = 403) => out(status, page(u.hostname, `<h1>${esc(u.hostname)}</h1><p class="hint">${esc(why)}.</p><p class="hint">${esc(u.href)}</p>`));
+  try {
+    for (let hop = 0; hop < 6; hop++) {
+      const why = await allowed(u, lookup);
+      if (why) return refuse(why);
+      const r = await fetchImpl(u.href, { redirect: 'manual', signal: AbortSignal.timeout(12000),
+        headers: { 'User-Agent': 'Mozilla/5.0 (compatible; Webship Workspace reader)', Accept: 'text/html,text/plain;q=0.9' } });
+      if (r.status >= 300 && r.status < 400 && r.headers.get('location')) { u = new URL(r.headers.get('location'), u); continue; }
+      if (!r.ok) return refuse(`that page answered ${r.status}`, r.status === 404 ? 404 : 502);
+      const type = String(r.headers.get('content-type') || '');
+      const body = (await r.text()).slice(0, MAX_BYTES);
+      const crumb = `<p class="hint">${esc(u.hostname)}${esc(u.pathname.length > 1 ? u.pathname.slice(0, 90) : '')}</p>`;
+      if (/text\/plain/.test(type)) return out(200, page(u.pathname.split('/').pop() || u.hostname, `${crumb}<pre>${esc(body.slice(0, 200000))}</pre>`));
+      if (!/text\/html|application\/xhtml/.test(type)) return refuse('that is not a page the reader can show', 415);
+      if (isChallenge(body)) return refuse(`${u.hostname} only opens in a real browser tab`);
+      const title = ((body.match(/<title[^>]*>([\s\S]{1,2000}?)<\/title>/i) || [])[1] || u.hostname).replace(/\s+/g, ' ').replace(/&amp;/g, '&').trim().slice(0, 140);
+      return out(200, page(title, `${crumb}${cleanHtml(body, u.href)}`));
+    }
+    return refuse('too many redirects');
+  } catch (_) { return refuse('that page could not be reached', 502); }
+}
+
+module.exports = { serveReader, allowed, privateIp, cleanHtml, isChallenge, readerLink };

package/repo-blobs.js

@@ -0,0 +1,66 @@
+/**
+ * Git blob identity, cheap on both sides.
+ *
+ * Remote: ONE recursive Trees call per repo@ref answers every "what is in the
+ * repo and at which sha" question a page has — cached for 5 minutes AND
+ * single-flighted, so ten rows rendering during a cold cache still cost one
+ * network call, not ten. `null` means unreachable, and is cached too: an
+ * unauthenticated container must not retry on every render.
+ *
+ * Local: sha1("blob <size>\0" + content), memoized by (size, mtime) — a page
+ * of fifty unchanged files hashes nothing on re-render.
+ */
+
+const fs = require('fs');
+const crypto = require('crypto');
+
+const { run } = require('./jobs');
+const { githubEnv } = require('./github-auth');
+
+const TTL_MS = 5 * 60 * 1000;
+const _trees = new Map(); // `${repo}@${ref}` -> { at, promise }
+
+/** Map<path, blob sha> for the whole repo at ref, or null when unreachable. */
+function repoTree(repo, ref) {
+  const key = `${repo}@${ref}`;
+  const hit = _trees.get(key);
+  if (hit && Date.now() - hit.at < TTL_MS) return hit.promise;
+  const promise = (async () => {
+    const res = await run('gh',
+      ['api', `repos/${repo}/git/trees/${encodeURIComponent(ref)}?recursive=1`,
+        '--jq', '.tree[] | select(.type=="blob") | .path + " " + .sha'],
+      process.env.HOME, { timeoutMs: 30 * 1000, env: githubEnv() });
+    if (res.code !== 0) return null;
+    const map = new Map();
+    for (const line of String(res.stdout || '').split('\n')) {
+      const sp = line.lastIndexOf(' ');
+      if (sp < 1) continue;
+      map.set(line.slice(0, sp), line.slice(sp + 1));
+    }
+    return map;
+  })();
+  _trees.set(key, { at: Date.now(), promise });
+  return promise;
+}
+
+function invalidateRepoTree(repo) {
+  for (const key of _trees.keys()) {
+    if (key.startsWith(`${repo}@`)) _trees.delete(key);
+  }
+}
+
+const _blobMemo = new Map(); // file -> { size, mtimeMs, sha }
+
+/** git's blob sha for a local file. Throws like fs does when it cannot be read. */
+function gitBlobSha(file) {
+  const st = fs.statSync(file);
+  const hit = _blobMemo.get(file);
+  if (hit && hit.size === st.size && hit.mtimeMs === st.mtimeMs) return hit.sha;
+  const buf = fs.readFileSync(file);
+  const sha = crypto.createHash('sha1')
+    .update(`blob ${buf.length}\0`).update(buf).digest('hex');
+  _blobMemo.set(file, { size: st.size, mtimeMs: st.mtimeMs, sha });
+  return sha;
+}
+
+module.exports = { repoTree, invalidateRepoTree, gitBlobSha };