factoryhub 0.1.0 → 0.1.1

package/packages/engine/lib/node.mjs

@@ -1,8 +1,9 @@
 // Node.js helpers around the engine: read a factory from a folder, a bundle
 // file or an address, and write one back to disk. The engine itself stays
 // free of file system access so the same code runs in a browser.
-import { readFile, writeFile, readdir, mkdir, lstat } from 'node:fs/promises';
-import { existsSync } from 'node:fs';
+import * as fsp from 'node:fs/promises';
+import { readFile, mkdir, lstat, open, unlink } from 'node:fs/promises';
+import { existsSync, constants } from 'node:fs';
 import { join, relative, resolve, dirname, sep } from 'node:path';
 import { fileURLToPath } from 'node:url';
 import { ALLOWED_PATH, fromEnvelope, normalizeFiles } from '../src/factory-engine.mjs';
@@ -18,58 +19,98 @@ const GENERATED = new Set(['RUNNING.md', 'run.mjs', 'factory-engine.mjs', 'autho
 /** The house factories that ship in this repository. */
 export const HOUSE_FACTORIES = resolve(dirname(fileURLToPath(import.meta.url)), '../../../factories');
 
+// At most `size` tasks given to the returned function run at once; a task
+// that ends hands its place to the next one waiting.
+function limiter(size) {
+  let active = 0;
+  const waiting = [];
+  return async (task) => {
+    if (active < size) active += 1;
+    else await new Promise((resolve) => waiting.push(resolve));
+    try { return await task(); } finally {
+      const next = waiting.shift();
+      if (next) next();
+      else active -= 1;
+    }
+  };
+}
+
+// File system calls of every folder being read share this limit, so reading
+// many factories at once never opens more files than this.
+const io = limiter(64);
+
 /**
  * Reads the files of a factory folder, as the engine admits them.
  * `skipped` lists files that look meant for the factory but are not part of it.
+ * Every directory and file is read at the same time; the results are then
+ * taken in the order a depth-first walk meets them, so `skipped` and the
+ * first error are the same as when they were read one by one. `fs` is for
+ * tests.
  */
-export async function readFolder(root) {
-  const files = {};
-  const skipped = [];
+export async function readFolder(root, { fs = fsp } = {}) {
   const walk = async (dir) => {
-    const entries = await readdir(dir, { withFileTypes: true });
+    const entries = await io(() => fs.readdir(dir, { withFileTypes: true }));
     entries.sort((a, b) => a.name.localeCompare(b.name));
-    for (const entry of entries) {
+    return Promise.all(entries.map(async (entry) => {
       const { name } = entry;
-      if (name.startsWith('.')) continue;
+      if (name.startsWith('.')) return null;
       const path = join(dir, name);
       const rel = relative(root, path).split(sep).join('/');
-      if (entry.isSymbolicLink()) {
-        if (dir !== root || SOURCE_FOLDERS.has(name) || ALLOWED_PATH.test(rel)) throw new Error(`Factory source cannot contain a symbolic link: ${rel}`);
-        continue;
+      try {
+        if (entry.isSymbolicLink()) {
+          if (dir !== root || SOURCE_FOLDERS.has(name) || ALLOWED_PATH.test(rel)) throw new Error(`Factory source cannot contain a symbolic link: ${rel}`);
+          return null;
+        }
+        if (entry.isDirectory()) return SOURCE_FOLDERS.has(name) || dir !== root ? { items: await walk(path) } : null;
+        if (dir === root && GENERATED.has(name)) return null;
+        if (ALLOWED_PATH.test(rel)) return { rel, content: await io(() => fs.readFile(path, 'utf8')) };
+        return dir !== root || /\.(md|ya?ml)$/i.test(name) ? { skipped: rel } : null;
+      } catch (error) {
+        return { error };
       }
-      if (entry.isDirectory()) {
-        if (SOURCE_FOLDERS.has(name) || dir !== root) await walk(path);
-        continue;
-      }
-      if (dir === root && GENERATED.has(name)) continue;
-      if (ALLOWED_PATH.test(rel)) files[rel] = await readFile(path, 'utf8');
-      else if (dir !== root || /\.(md|ya?ml)$/i.test(name)) skipped.push(rel);
+    }));
+  };
+  const files = {};
+  const skipped = [];
+  const take = (items) => {
+    for (const item of items) {
+      if (!item) continue;
+      if (item.error) throw item.error;
+      if (item.items) take(item.items);
+      else if (item.skipped) skipped.push(item.skipped);
+      else files[item.rel] = item.content;
     }
   };
-  await walk(root);
+  take(await walk(root));
   return { files: normalizeFiles(files), skipped };
 }
 
 const isUrl = (value) => /^https?:\/\//i.test(String(value));
 
+/** How long reading a bundle from an address may take, as for the hub client. */
+export const BUNDLE_TIMEOUT_MS = 30000;
+
 /**
  * Resolves what a person typed to a factory: a folder, a bundle `.json` file,
  * an http(s) address of a bundle, or the id of a house factory.
- * Returns { files, leftOut, source: { kind, location, id } }.
+ * Returns { files, leftOut, source: { kind, location, id } }. An address that
+ * does not answer, or does not finish its answer, within `timeoutMs` fails.
  */
-export async function readFactory(target, { cwd = process.cwd(), fetchImpl = fetch, headers = {}, houseDir = HOUSE_FACTORIES } = {}) {
+export async function readFactory(target, { cwd = process.cwd(), fetchImpl = fetch, headers = {}, houseDir = HOUSE_FACTORIES, timeoutMs = BUNDLE_TIMEOUT_MS } = {}) {
   if (!target) throw usage('Name a factory: a folder, a bundle .json file, an address, or a house factory id.');
   if (isUrl(target)) {
-    const response = await fetchImpl(target, { headers: { accept: 'application/json', ...headers } });
+    const timedOut = (error) => (error?.name === 'TimeoutError' ? new Error(`${target} did not answer within ${timeoutMs / 1000} seconds.`) : error);
+    // The signal also ends a body that stops arriving.
+    const response = await fetchImpl(target, { headers: { accept: 'application/json', ...headers }, signal: AbortSignal.timeout(timeoutMs) }).catch((error) => { throw timedOut(error); });
     if (!response.ok) throw new Error(`${target} answered with status ${response.status}.`);
-    const read = await fromEnvelope(await response.json());
+    const read = await fromEnvelope(await response.json().catch((error) => { throw timedOut(error); }));
     if (read.problems.length) throw new Error(`That bundle could not be read:\n${read.problems.map((problem) => `  - ${problem}`).join('\n')}`);
     return { files: read.files, leftOut: read.leftOut, source: { kind: 'url', location: target } };
   }
   const path = resolve(cwd, target);
   if (existsSync(path) && (await lstat(path)).isFile()) {
     if (!path.endsWith('.json')) throw usage(`${target} is a file. Point at a factory folder or a bundle .json file.`);
-    const read = await fromEnvelope(JSON.parse(await readFile(path, 'utf8')));
+    const read = await fromEnvelope(JSON.parse(await fsp.readFile(path, 'utf8')));
     if (read.problems.length) throw new Error(`That bundle could not be read:\n${read.problems.map((problem) => `  - ${problem}`).join('\n')}`);
     return { files: read.files, leftOut: read.leftOut, source: { kind: 'bundle', location: path } };
   }
@@ -89,23 +130,110 @@ export async function readFactory(target, { cwd = process.cwd(), fetchImpl = fet
 /** The ids of the factory folders directly inside `dir`, sorted. */
 export async function listFactoryDirs(dir = HOUSE_FACTORIES) {
   if (!existsSync(dir)) return [];
-  const entries = await readdir(dir, { withFileTypes: true });
+  const entries = await fsp.readdir(dir, { withFileTypes: true });
   return entries.filter((entry) => entry.isDirectory() && existsSync(join(dir, entry.name, 'factory.yaml'))).map((entry) => entry.name).sort();
 }
 
+const lstatOrNull = async (path) => {
+  try { return await lstat(path); } catch (error) {
+    if (error.code === 'ENOENT') return null;
+    throw error;
+  }
+};
+// Opens a file for writing without following a symbolic link where the
+// platform can refuse one (O_NOFOLLOW); every folder above it is checked first.
+const WRITE_FLAGS = constants.O_WRONLY | constants.O_CREAT | constants.O_TRUNC | (constants.O_NOFOLLOW || 0);
+
+/**
+ * The folder files are written into, made if it is missing. A folder that is
+ * itself a symbolic link is refused: writing through it would land elsewhere.
+ */
+async function writableRoot(dir) {
+  const root = resolve(dir);
+  const info = await lstatOrNull(root);
+  if (info?.isSymbolicLink()) throw new Error(`Refusing to write into ${root}: it is a symbolic link.`);
+  if (info && !info.isDirectory()) throw new Error(`Refusing to write into ${root}: it is not a folder.`);
+  if (!info) await mkdir(root, { recursive: true });
+  return root;
+}
+
+/**
+ * The absolute path for one relative file under `root`, with every folder on
+ * the way made or checked: none may be a symbolic link, and the file itself,
+ * if it exists, must be a plain file.
+ */
+async function safeTarget(root, path) {
+  const target = resolve(root, path);
+  if (!target.startsWith(root + sep)) throw new Error(`Refusing to write outside ${root}: ${path}`);
+  const parts = relative(root, target).split(sep);
+  let current = root;
+  for (const part of parts.slice(0, -1)) {
+    current = join(current, part);
+    const info = await lstatOrNull(current);
+    if (!info) { await mkdir(current); continue; }
+    if (info.isSymbolicLink()) throw new Error(`Refusing to write "${path}": ${relative(root, current).split(sep).join('/')} is a symbolic link.`);
+    if (!info.isDirectory()) throw new Error(`Refusing to write "${path}": ${relative(root, current).split(sep).join('/')} is not a folder.`);
+  }
+  const info = await lstatOrNull(target);
+  if (info?.isSymbolicLink()) throw new Error(`Refusing to write "${path}": it is a symbolic link.`);
+  if (info && !info.isFile()) throw new Error(`Refusing to write "${path}": it is not a plain file.`);
+  return target;
+}
+
+async function writeOne(target, content) {
+  const handle = await open(target, WRITE_FLAGS, 0o644);
+  try { await handle.writeFile(content); } finally { await handle.close(); }
+}
+
 /**
  * Writes bundle files under `dir`. Every path must be one the engine admits,
- * so a bundle from elsewhere cannot write outside its folder.
+ * so a bundle from elsewhere cannot write outside its folder, and no file is
+ * written through a symbolic link.
  */
 export async function writeFiles(dir, files, { allow = (path) => ALLOWED_PATH.test(path) } = {}) {
-  const root = resolve(dir);
-  for (const [path, content] of Object.entries(files)) {
-    if (!allow(path)) throw new Error(`Refusing to write "${path}": it is not a factory file.`);
-    const target = resolve(root, path);
-    if (!target.startsWith(root + sep)) throw new Error(`Refusing to write outside ${root}: ${path}`);
-    await mkdir(dirname(target), { recursive: true });
-    await writeFile(target, content);
+  for (const path of Object.keys(files)) if (!allow(path)) throw new Error(`Refusing to write "${path}": it is not a factory file.`);
+  const root = await writableRoot(dir);
+  for (const [path, content] of Object.entries(files)) await writeOne(await safeTarget(root, path), content);
+}
+
+/** The record `pull` keeps of the files it wrote, so a later pull removes only those. */
+export const PULL_MANIFEST = '.factoryhub-pull.json';
+
+/**
+ * Writes a pulled factory into `dir` and records what it wrote in
+ * PULL_MANIFEST. With `replace`, files a previous pull wrote (as its
+ * manifest lists them) that the new version no longer has are removed; no
+ * other file is ever deleted. Returns { removed }.
+ */
+export async function writePulled(dir, files, { replace = false, record = {} } = {}) {
+  const root = await writableRoot(dir);
+  const removed = [];
+  if (replace) {
+    const listed = await readPullManifest(root);
+    for (const path of listed) {
+      if (files[path] !== undefined || !ALLOWED_PATH.test(path)) continue;
+      const target = await safeTarget(root, path).catch(() => null);
+      if (!target) continue;
+      const info = await lstatOrNull(target);
+      if (!info?.isFile()) continue;
+      await unlink(target);
+      removed.push(path);
+    }
   }
+  await writeFiles(root, files);
+  const manifest = await safeTarget(root, PULL_MANIFEST);
+  await writeOne(manifest, `${JSON.stringify({ format: 'factoryhub.pull/v1', ...record, files: Object.keys(files) }, null, 2)}\n`);
+  return { removed };
+}
+
+async function readPullManifest(root) {
+  const path = join(root, PULL_MANIFEST);
+  const info = await lstatOrNull(path);
+  if (!info?.isFile()) return [];
+  try {
+    const parsed = JSON.parse(await readFile(path, 'utf8'));
+    return Array.isArray(parsed?.files) ? parsed.files.filter((file) => typeof file === 'string') : [];
+  } catch { return []; }
 }
 
 /** An error that means the person asked for something unclear: exit code 2. */

package/packages/engine/lib/providers.mjs

@@ -25,8 +25,11 @@ const trim = (url) => String(url).replace(/\/+$/, '');
  *   carry a provider key away.
  * - `model` falls back to FACTORYHUB_MODEL, then to a default for hosted
  *   providers. Local servers need a model name.
+ * - `effort` (anthropic only) sets output_config.effort; `maxOutput` the
+ *   room each call leaves for thinking and the answer; `timeoutMs` how long
+ *   one call may take.
  */
-export function chooseProvider({ provider, model, baseUrl, apiKeyEnv, env = process.env, fetchImpl = fetch } = {}) {
+export function chooseProvider({ provider, model, baseUrl, apiKeyEnv, effort, maxOutput, timeoutMs, env = process.env, fetchImpl = fetch } = {}) {
   let kind = provider ? String(provider).toLowerCase() : null;
   if (kind && !PROVIDERS.includes(kind)) throw problem(`Unknown provider "${provider}". Use one of: ${PROVIDERS.join(', ')}.`);
   if (!kind) {
@@ -47,6 +50,8 @@ export function chooseProvider({ provider, model, baseUrl, apiKeyEnv, env = proc
   if (!address) throw problem('An OpenAI-compatible server needs a base URL.');
   // The provider's own key for its own address; otherwise only a key the person named.
   const apiKey = apiKeyEnv ? keyFrom(apiKeyEnv) : KEY_OF[kind] && address === OFFICIAL[kind] ? keyFrom(KEY_OF[kind]) : '';
-  if (kind === 'anthropic') return anthropicProvider({ apiKey, model: chosen, fetchImpl, baseUrl: address });
-  return openaiProvider({ apiKey, model: chosen, fetchImpl, baseUrl: address });
+  if (effort && kind !== 'anthropic') throw problem('Effort is set only for the anthropic provider.');
+  const extra = { ...(timeoutMs ? { timeoutMs } : {}) };
+  if (kind === 'anthropic') return anthropicProvider({ apiKey, model: chosen, fetchImpl, baseUrl: address, effort: effort || null, ...(maxOutput ? { maxOutput } : {}), ...extra });
+  return openaiProvider({ apiKey, model: chosen, fetchImpl, baseUrl: address, ...extra });
 }

package/packages/engine/lib/run.mjs

@@ -9,6 +9,58 @@ import { canonicalFiles } from './catalog.mjs';
 
 export const USER_AGENT = 'Mozilla/5.0 (compatible; FactoryRunner/1.0)';
 
+/** How much of a page is read, as on the factoryhub site: the rest is never downloaded. */
+export const PAGE_BYTES = 1500000;
+const CUT_NOTE = `[Page cut here: only the first ${PAGE_BYTES.toLocaleString('en-US')} bytes of the page were read]`;
+
+/**
+ * Reads a response body up to `max` bytes, then stops the download.
+ * Returns { text, cut }; the text is decoded as UTF-8, as response.text() does.
+ */
+export async function readCapped(response, max = PAGE_BYTES) {
+  const reader = response.body?.getReader();
+  const chunks = [];
+  let total = 0;
+  let cut = false;
+  while (reader) {
+    const { done, value } = await reader.read();
+    if (done) break;
+    chunks.push(value);
+    total += value.byteLength;
+    if (total >= max) {
+      // A page that ends exactly at the limit was not cut.
+      cut = total > max || !(await reader.read()).done;
+      await reader.cancel().catch(() => {});
+      break;
+    }
+  }
+  const bytes = new Uint8Array(Math.min(total, max));
+  let offset = 0;
+  for (const chunk of chunks) {
+    const part = chunk.subarray(0, bytes.length - offset);
+    bytes.set(part, offset);
+    offset += part.byteLength;
+  }
+  return { text: new TextDecoder('utf-8').decode(bytes), cut };
+}
+
+/**
+ * The text an agent gets from a page response: only text, HTML, XML and JSON
+ * are read, and at most PAGE_BYTES of them. A cut page says it was cut, so
+ * the run records that only part of it was read.
+ */
+export async function pageText(response, url) {
+  const type = (response.headers.get('content-type') || '').toLowerCase();
+  if (type && !/text\/|json|xml/.test(type)) {
+    await response.body?.cancel().catch(() => {});
+    throw new Error(`the address is a ${type.split(';')[0].trim()} file, not a page`);
+  }
+  const { text, cut } = await readCapped(response);
+  const page = htmlToText(text, url);
+  const note = cut && !/\[Page cut here: [^\]]*\]$/.test(page) ? CUT_NOTE : '';
+  return `Address: ${url}\n\n${[page, note].filter(Boolean).join('\n\n')}`;
+}
+
 const v4 = (ip) => ip.split('.').map(Number);
 /** True for loopback, private, link-local, shared, reserved and multicast addresses. */
 export function isPrivateIp(ip) {
@@ -46,7 +98,7 @@ export function nodeHost({ provider, emit = () => {}, fetchImpl = fetch, resolve
         const response = await fetchImpl(url, { redirect: 'manual', headers: { 'user-agent': USER_AGENT, accept: 'text/html,*/*;q=0.5' }, signal: AbortSignal.timeout(20000) });
         if (response.status >= 300 && response.status < 400) { url = new URL(response.headers.get('location') || '', url).href; continue; }
         if (!response.ok) throw new Error(`the site answered with status ${response.status}`);
-        return `Address: ${url}\n\n${htmlToText(await response.text(), url)}`;
+        return pageText(response, url);
       }
       throw new Error('the page redirects too many times');
     },
@@ -58,16 +110,25 @@ export function nodeHost({ provider, emit = () => {}, fetchImpl = fetch, resolve
  * Prepares a run: parses the factory, refuses one that cannot run here, and
  * checks the inputs before any model is called.
  */
-export function prepareRun({ files, inputs = {}, id, runner = { mode: 'local' } }) {
-  const parsed = parseBundle(canonicalFiles(files));
+export function prepareRun({ files, inputs = {}, id, runner = { mode: 'local' }, toolSources = null, features = [] }) {
+  // `features`: experimental execution features the caller opts in to (tool sources a flow declares).
+  const parsed = parseBundle(canonicalFiles(files), features.length ? { features } : undefined);
   if (!parsed.ok) return { ok: false, problems: parsed.errors };
-  if (!parsed.factory.runnable) {
-    return { ok: false, problems: parsed.factory.blockers.length ? parsed.factory.blockers : ['This factory has no flow.yaml, so there are no steps to run. Follow agents/foreman/agent.md in an agent instead.'] };
+  // Experimental, opt-in: tool sources (createToolSources) may supply tools
+  // that steps name. Without them nothing here changes.
+  const admitted = toolSources ? toolSources.admit(parsed.factory, { exact: features.includes('tool-sources') }) : { factory: parsed.factory, problems: [] };
+  const { factory } = admitted;
+  if (!factory.runnable) {
+    return { ok: false, problems: factory.blockers.length ? factory.blockers : ['This factory has no flow.yaml, so there are no steps to run. Follow agents/foreman/agent.md in an agent instead.'] };
+  }
+  const checked = checkInputs(factory, inputs);
+  if (!checked.ok) return { ok: false, problems: checked.problems, factory };
+  const state = newRun({ id: id || `local-${Date.now().toString(36)}`, factory, inputs: checked.inputs, runner });
+  if (toolSources) {
+    state.experimental = [toolSources.feature];
+    if (toolSources.records) state.toolSourceRecords = toolSources.records();
   }
-  const checked = checkInputs(parsed.factory, inputs);
-  if (!checked.ok) return { ok: false, problems: checked.problems, factory: parsed.factory };
-  const state = newRun({ id: id || `local-${Date.now().toString(36)}`, factory: parsed.factory, inputs: checked.inputs, runner });
-  return { ok: true, state, factory: parsed.factory };
+  return { ok: true, state, factory };
 }
 
 /**

package/packages/engine/lib/text.mjs

@@ -0,0 +1,5 @@
+// Small text checks with no imports, so a caller that needs one of them does
+// not load the hub client or the engine with it.
+
+/** True when `text` holds a control character (U+0000 to U+001F, or U+007F). */
+export const hasControl = (text) => [...String(text)].some((char) => char.charCodeAt(0) < 0x20 || char.charCodeAt(0) === 0x7f);

package/packages/engine/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@factoryhub/engine",
-  "version": "0.1.0",
+  "version": "0.1.1",
   "description": "The factory engine: parses factory bundles, runs their flows with your model, and writes and verifies receipts. No dependencies.",
   "license": "Apache-2.0",
   "type": "module",
@@ -22,12 +22,18 @@
     "./diagnostics": "./src/diagnostics.mjs",
     "./markdown": "./src/markdown.mjs",
     "./package": "./src/package.mjs",
+    "./mcp-client": "./src/mcp-client.mjs",
+    "./tool-sources": "./src/tool-sources.mjs",
+    "./flow-sources": "./src/flow-sources.mjs",
+    "./nango": "./lib/nango.mjs",
     "./node": "./lib/node.mjs",
     "./skill": "./lib/skill.mjs",
     "./hub": "./lib/hub.mjs",
     "./catalog": "./lib/catalog.mjs",
     "./run": "./lib/run.mjs",
-    "./providers": "./lib/providers.mjs"
+    "./providers": "./lib/providers.mjs",
+    "./credentials": "./lib/credentials.mjs",
+    "./hosted": "./lib/hosted.mjs"
   },
   "files": [
     "src",