@sema-agent/connectors 0.1.0 → 0.3.0

package/dist/pinned-upstream.js

@@ -0,0 +1,347 @@
+import { spawn } from 'node:child_process';
+import { createHash, randomBytes } from 'node:crypto';
+import { chmodSync, closeSync, constants, fchmodSync, fstatSync, lstatSync, mkdirSync, openSync, readFileSync, renameSync, rmSync, statSync } from 'node:fs';
+import { get as httpsGet } from 'node:https';
+import { constants as osConstants } from 'node:os';
+import { dirname, isAbsolute, join, relative, sep } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { cliOutput, listed } from './cli-kit.js';
+import { given } from './config-dir.js';
+import { CONNECTOR_NAME } from './connector-name.js';
+import { entryOf } from './release-archive.js';
+import { HANDLE_NAME, openFileSecrets } from './secrets.js';
+import { writeAll } from './write-all.js';
+const CONNECTOR_DIR = 'CONNECTOR_DIR';
+const BIN_DIR = 'bin';
+const DIRECTORY_MODE = 0o700;
+const BINARY_MODE = 0o755;
+const ARCHIVE_MODE = 0o644;
+const DOWNLOAD_LIMIT = 256 * 1024 * 1024;
+const DOWNLOAD_IDLE_MS = 120_000;
+const REDIRECT_LIMIT = 5;
+const SIGNALS = ['SIGTERM', 'SIGINT', 'SIGHUP'];
+const VERSION = /^[0-9]+\.[0-9]+\.[0-9]+$/;
+const PLATFORM = /^[a-z0-9]+-[a-z0-9]+$/;
+const FILE = /^[A-Za-z0-9][A-Za-z0-9._-]*$/;
+const SHA256 = /^[0-9a-f]{64}$/;
+const platformKey = () => `${process.platform}-${process.arch}`;
+const isObject = (v) => v !== null && typeof v === 'object' && !Array.isArray(v);
+const keysAre = (v, keys) => Object.keys(v).sort().join('\0') === [...keys].sort().join('\0');
+const sha256 = (bytes) => createHash('sha256').update(bytes).digest('hex');
+const below = (dir, file) => {
+    if (!isAbsolute(file))
+        return false;
+    const r = relative(dir, file);
+    return r !== '' && !isAbsolute(r) && r.split(sep)[0] !== '..';
+};
+const isRegularFile = (path) => {
+    try {
+        return lstatSync(path).isFile();
+    }
+    catch {
+        return false;
+    }
+};
+function regularFile(path) {
+    let fd;
+    try {
+        fd = openSync(path, constants.O_RDONLY | constants.O_NOFOLLOW);
+    }
+    catch {
+        return null;
+    }
+    try {
+        return fstatSync(fd).isFile() ? readFileSync(fd) : null;
+    }
+    catch {
+        return null;
+    }
+    finally {
+        closeSync(fd);
+    }
+}
+function fetchHttps(url, redirects) {
+    return new Promise((resolve) => {
+        let settled = false;
+        const done = (r) => {
+            if (settled)
+                return;
+            settled = true;
+            resolve(r);
+        };
+        const req = httpsGet(url, (res) => {
+            const status = res.statusCode ?? 0;
+            const location = res.headers.location;
+            if (status >= 300 && status < 400 && location !== undefined) {
+                res.resume();
+                let next;
+                try {
+                    next = new URL(location, url);
+                }
+                catch {
+                    return done({ ok: false, problem: 'the download was redirected to something that is not a URL' });
+                }
+                if (redirects === 0)
+                    return done({ ok: false, problem: `the download was redirected more than ${REDIRECT_LIMIT} times` });
+                if (next.protocol !== 'https:' || next.username !== '' || next.password !== '')
+                    return done({ ok: false, problem: 'the download was redirected away from https' });
+                return void fetchHttps(next, redirects - 1).then(done);
+            }
+            if (status !== 200) {
+                res.resume();
+                return done({ ok: false, problem: `the download answered HTTP ${status}` });
+            }
+            const parts = [];
+            let size = 0;
+            res.on('data', (d) => {
+                size += d.length;
+                if (size > DOWNLOAD_LIMIT) {
+                    req.destroy();
+                    done({ ok: false, problem: `the archive is larger than ${DOWNLOAD_LIMIT} bytes` });
+                }
+                else
+                    parts.push(d);
+            });
+            res.on('end', () => done({ ok: true, bytes: Buffer.concat(parts) }));
+            res.on('error', () => done({ ok: false, problem: 'the download broke off' }));
+        });
+        req.setTimeout(DOWNLOAD_IDLE_MS, () => req.destroy(new Error(`no data for ${DOWNLOAD_IDLE_MS / 1000} s`)));
+        req.on('error', (e) => done({ ok: false, problem: `the download failed (${e.message})` }));
+    });
+}
+async function fetchArchive(url) {
+    if (url.protocol === 'https:')
+        return fetchHttps(url, REDIRECT_LIMIT);
+    const path = fileURLToPath(url);
+    try {
+        const st = statSync(path);
+        if (!st.isFile())
+            return { ok: false, problem: `${path} is not a file` };
+        if (st.size > DOWNLOAD_LIMIT)
+            return { ok: false, problem: `the archive is larger than ${DOWNLOAD_LIMIT} bytes` };
+        return { ok: true, bytes: readFileSync(path) };
+    }
+    catch {
+        return { ok: false, problem: `${path} cannot be read` };
+    }
+}
+export function pinnedUpstream(options) {
+    const { connector, upstream, lockFile } = options ?? {};
+    if (typeof connector !== 'string' || !CONNECTOR_NAME.test(connector))
+        throw new Error('connector must be lowercase letters and digits in hyphen-separated words');
+    if (typeof upstream !== 'string' || !FILE.test(upstream))
+        throw new Error('upstream must be a file name: letters, digits, dots, underscores and hyphens, starting with a letter or digit');
+    if (!(lockFile instanceof URL) || lockFile.protocol !== 'file:')
+        throw new Error('lockFile must be a file URL');
+    const handles = options.handles;
+    if (!Array.isArray(handles) || !handles.every((h) => typeof h === 'string' && HANDLE_NAME.test(h)) || new Set(handles).size !== handles.length)
+        throw new Error('handles must list handle names, each once');
+    const { program, say, complain } = cliOutput(connector);
+    const fileVars = handles.map((h) => `${h}_FILE`);
+    const binaryOf = (connectorDir) => join(connectorDir, BIN_DIR, upstream);
+    function pinnedBuild() {
+        const refuse = (problem) => ({ ok: false, problem: `binaries.lock.json ${problem}` });
+        const isBuild = (v) => isObject(v) && keysAre(v, ['file', 'sha256']) && typeof v.file === 'string' && FILE.test(v.file) && v.file !== upstream && typeof v.sha256 === 'string' && SHA256.test(v.sha256);
+        let doc;
+        try {
+            doc = JSON.parse(readFileSync(lockFile, 'utf8'));
+        }
+        catch {
+            return refuse('cannot be read as JSON');
+        }
+        if (!isObject(doc) || !keysAre(doc, [upstream]))
+            return refuse(`must hold exactly one entry, "${upstream}"`);
+        const entry = doc[upstream];
+        if (!isObject(entry) || !keysAre(entry, ['version', 'source', 'platforms']))
+            return refuse('must give exactly version, source and platforms');
+        const { version, source, platforms } = entry;
+        if (typeof version !== 'string' || !VERSION.test(version))
+            return refuse('must give the version as <major>.<minor>.<patch>');
+        if (typeof source !== 'string' || !source.includes('{file}'))
+            return refuse('must give the source as a URL template naming {file}');
+        if (!isObject(platforms) || !Object.entries(platforms).every(([k, v]) => PLATFORM.test(k) && isBuild(v)))
+            return refuse('must map each platform to {"file": <archive name>, "sha256": <64 hex digits>}');
+        const key = platformKey();
+        const build = Object.hasOwn(platforms, key) ? platforms[key] : undefined;
+        if (!isBuild(build))
+            return refuse(`pins no ${upstream} build for ${key}`);
+        return { ok: true, lock: { version, source, platforms: platforms }, build };
+    }
+    function sourceOf(lock, build) {
+        let url;
+        try {
+            url = new URL(lock.source.replaceAll('{version}', lock.version).replaceAll('{file}', build.file));
+        }
+        catch {
+            return { ok: false, problem: 'the source in binaries.lock.json is not a URL' };
+        }
+        if (url.protocol === 'https:' && url.username === '' && url.password === '')
+            return { ok: true, url };
+        if (url.protocol === 'file:' && (url.host === '' || url.host === 'localhost'))
+            return { ok: true, url };
+        return { ok: false, problem: 'the source in binaries.lock.json must be an https or file URL (an https URL without credentials)' };
+    }
+    function installedProblem(connectorDir, build) {
+        const dir = join(connectorDir, BIN_DIR);
+        let st;
+        try {
+            st = lstatSync(dir);
+        }
+        catch {
+            return `${upstream} is not installed`;
+        }
+        if (!st.isDirectory() || (st.mode & 0o777) !== DIRECTORY_MODE)
+            return 'the bin directory of the connector is not a 0700 directory';
+        const archive = regularFile(join(dir, build.file));
+        if (archive === null)
+            return `${upstream} is not installed (the pinned archive is not there)`;
+        if (sha256(archive) !== build.sha256)
+            return 'the archive there is not the one binaries.lock.json pins';
+        const entry = entryOf(archive, upstream);
+        if (!entry.ok)
+            return `the pinned archive ${entry.problem}`;
+        const binary = binaryOf(connectorDir);
+        const bytes = regularFile(binary);
+        if (bytes === null || !bytes.equals(entry.bytes))
+            return `${upstream} there is not the entry of the pinned archive`;
+        if ((lstatSync(binary).mode & 0o777) !== BINARY_MODE)
+            return `${upstream} there is not mode 0755`;
+        return null;
+    }
+    function writeAtomically(path, bytes, mode) {
+        const tmp = join(dirname(path), `.${upstream}.${randomBytes(6).toString('hex')}.tmp`);
+        try {
+            const fd = openSync(tmp, 'wx', mode);
+            try {
+                fchmodSync(fd, mode);
+                writeAll(fd, bytes);
+            }
+            finally {
+                closeSync(fd);
+            }
+            renameSync(tmp, path);
+        }
+        catch (e) {
+            rmSync(tmp, { force: true });
+            throw e;
+        }
+    }
+    async function install(configDir) {
+        const pinned = pinnedBuild();
+        if (!pinned.ok) {
+            complain(`${pinned.problem}; nothing was installed`);
+            return 1;
+        }
+        const { lock, build } = pinned;
+        const secrets = openFileSecrets({ configDir, connector });
+        const before = secrets.list();
+        if (!before.ok) {
+            complain(`${before.problem}; nothing was installed`);
+            return 1;
+        }
+        const connectorDir = secrets.directory;
+        const binDir = join(connectorDir, BIN_DIR);
+        if (installedProblem(connectorDir, build) === null) {
+            say(`${upstream} ${lock.version} is already installed in ${binDir}; nothing changed`);
+            return 0;
+        }
+        const source = sourceOf(lock, build);
+        if (!source.ok) {
+            complain(`${source.problem}; nothing was installed`);
+            return 1;
+        }
+        const fetched = await fetchArchive(source.url);
+        if (!fetched.ok) {
+            complain(`${build.file} could not be fetched: ${fetched.problem}; nothing was installed`);
+            return 1;
+        }
+        const actual = sha256(fetched.bytes);
+        if (actual !== build.sha256) {
+            complain(`the fetched ${build.file} does not match binaries.lock.json (sha256 ${actual}, pinned ${build.sha256}); nothing was installed`);
+            return 1;
+        }
+        const entry = entryOf(fetched.bytes, upstream);
+        if (!entry.ok) {
+            complain(`the pinned archive ${build.file} ${entry.problem}; nothing was installed`);
+            return 1;
+        }
+        try {
+            mkdirSync(binDir, { recursive: true, mode: DIRECTORY_MODE });
+            const after = secrets.list();
+            if (!after.ok)
+                throw new Error(after.problem);
+            const st = lstatSync(binDir);
+            if (!st.isDirectory())
+                throw new Error('the bin directory of the connector is not a directory');
+            chmodSync(binDir, DIRECTORY_MODE);
+            writeAtomically(join(binDir, build.file), fetched.bytes, ARCHIVE_MODE);
+            writeAtomically(binaryOf(connectorDir), entry.bytes, BINARY_MODE);
+        }
+        catch (e) {
+            complain(`${upstream} was not installed: ${e instanceof Error ? e.message : 'unknown error'}`);
+            return 1;
+        }
+        const problem = installedProblem(connectorDir, build);
+        if (problem !== null) {
+            complain(`${upstream} was written but does not read back as the pinned build: ${problem}`);
+            return 1;
+        }
+        say(`installed ${upstream} ${lock.version} (${build.file}, sha256 ${build.sha256}) in ${binDir}`);
+        return 0;
+    }
+    function run(binary, args, env) {
+        return new Promise((resolve) => {
+            const child = spawn(binary, [...args], { stdio: 'inherit', env });
+            const forwarders = SIGNALS.map((signal) => [signal, () => void child.kill(signal)]);
+            for (const [signal, forward] of forwarders)
+                process.on(signal, forward);
+            const stopForwarding = () => {
+                for (const [signal, forward] of forwarders)
+                    process.off(signal, forward);
+            };
+            child.on('error', (e) => {
+                stopForwarding();
+                complain(`${upstream} could not be started (${e.code ?? e.message})`);
+                resolve(1);
+            });
+            child.on('exit', (code, signal) => {
+                stopForwarding();
+                if (signal !== null) {
+                    process.kill(process.pid, signal);
+                    setTimeout(() => resolve(128 + (osConstants.signals[signal] ?? 0)), 1000);
+                    return;
+                }
+                resolve(code ?? 1);
+            });
+        });
+    }
+    function start(env) {
+        const refuse = (problem) => ({ ok: false, problem });
+        const connectorDir = env[CONNECTOR_DIR];
+        if (!given(connectorDir) || !isAbsolute(connectorDir))
+            return refuse(`${CONNECTOR_DIR} does not name the connector's directory: start the connector with sema-connector run ${connector}`);
+        const paths = fileVars.map((v) => env[v]);
+        if (!paths.every(given))
+            return refuse(`connector ${connector} has no ${listed(handles)} ${handles.length === 1 ? 'handle' : 'handles'} to start ${upstream} with: store ${handles.length === 1 ? 'it' : 'them'} with ${program} auth`);
+        if (!paths.every((p) => below(connectorDir, p) && isRegularFile(p)))
+            return refuse(`${listed(fileVars)} must name files in the connector's directory (${CONNECTOR_DIR}), as the launcher hands ${handles.length === 1 ? 'it' : 'them'}: start the connector with sema-connector run ${connector}`);
+        const pinned = pinnedBuild();
+        if (!pinned.ok)
+            return refuse(pinned.problem);
+        const problem = installedProblem(connectorDir, pinned.build);
+        if (problem !== null)
+            return refuse(`${problem}: run ${program} install`);
+        const own = {};
+        for (const [k, v] of Object.entries(env))
+            if (v !== undefined && k !== CONNECTOR_DIR && !fileVars.includes(k))
+                own[k] = v;
+        const binary = binaryOf(connectorDir);
+        return {
+            ok: true,
+            connectorDir,
+            files: Object.fromEntries(handles.map((h, i) => [h, paths[i]])),
+            run: (args, extra) => run(binary, args, { ...own, ...extra }),
+        };
+    }
+    return { install, start };
+}

package/dist/release-archive.d.ts

@@ -0,0 +1,8 @@
+export type Entry = {
+    readonly ok: true;
+    readonly bytes: Buffer;
+} | {
+    readonly ok: false;
+    readonly problem: string;
+};
+export declare function entryOf(archive: Buffer, wanted: string): Entry;

package/dist/release-archive.js

@@ -0,0 +1,96 @@
+import { gunzipSync } from 'node:zlib';
+const BLOCK = 512;
+const UNPACKED_LIMIT = 512 * 1024 * 1024;
+function text(h, at, length) {
+    const raw = h.subarray(at, at + length);
+    const end = raw.indexOf(0);
+    return raw.subarray(0, end === -1 ? length : end).toString('utf8');
+}
+function octal(h, at, length) {
+    const t = text(h, at, length).trim();
+    return /^[0-7]+$/.test(t) ? parseInt(t, 8) : null;
+}
+function pax(body) {
+    const out = {};
+    for (let at = 0; at < body.length;) {
+        const space = body.indexOf(0x20, at);
+        const length = space === -1 ? NaN : Number(body.subarray(at, space).toString('latin1'));
+        if (!Number.isInteger(length) || length <= space - at || at + length > body.length)
+            return null;
+        const record = body.subarray(space + 1, at + length - 1).toString('utf8');
+        const eq = record.indexOf('=');
+        if (eq === -1)
+            return null;
+        const key = record.slice(0, eq);
+        const value = record.slice(eq + 1);
+        if (key === 'path')
+            out.path = value;
+        if (key === 'size') {
+            if (!/^[0-9]+$/.test(value))
+                return null;
+            out.size = Number(value);
+        }
+        at += length;
+    }
+    return out;
+}
+export function entryOf(archive, wanted) {
+    const refuse = (problem) => ({ ok: false, problem });
+    let tar;
+    try {
+        tar = gunzipSync(archive, { maxOutputLength: UNPACKED_LIMIT });
+    }
+    catch {
+        return refuse('is not a gzip stream within the size limit');
+    }
+    const found = [];
+    let next = {};
+    for (let at = 0;;) {
+        if (at === tar.length)
+            break;
+        if (at + BLOCK > tar.length)
+            return refuse('ends inside a header');
+        const h = tar.subarray(at, at + BLOCK);
+        if (h.every((b) => b === 0))
+            break;
+        let sum = 0;
+        for (let i = 0; i < BLOCK; i++)
+            sum += i >= 148 && i < 156 ? 0x20 : (h[i] ?? 0);
+        if (octal(h, 148, 8) !== sum)
+            return refuse('has a damaged header');
+        const type = h[156] === 0 ? '0' : String.fromCharCode(h[156] ?? 0);
+        const meta = type === 'x' || type === 'g' || type === 'L';
+        const own = octal(h, 124, 12);
+        const size = meta ? own : (next.size ?? own);
+        if (size === null)
+            return refuse('has a damaged header');
+        const start = at + BLOCK;
+        if (start + size > tar.length)
+            return refuse('ends inside an entry');
+        const body = tar.subarray(start, start + size);
+        at = start + Math.ceil(size / BLOCK) * BLOCK;
+        if (type === 'x') {
+            const records = pax(body);
+            if (records === null)
+                return refuse('has a damaged extended header');
+            next = { ...next, ...records };
+            continue;
+        }
+        if (type === 'L') {
+            next = { ...next, path: body.toString('utf8').replace(/\0+$/, '') };
+            continue;
+        }
+        if (type === 'g')
+            continue;
+        const prefix = text(h, 257, 6).startsWith('ustar') ? text(h, 345, 155) : '';
+        const name = next.path ?? (prefix === '' ? text(h, 0, 100) : `${prefix}/${text(h, 0, 100)}`);
+        next = {};
+        if ((type === '0' || type === '7') && name.replace(/^\.\//, '') === wanted)
+            found.push(Buffer.from(body));
+    }
+    if (found.length === 0)
+        return refuse(`holds no ${wanted}`);
+    if (found.length > 1)
+        return refuse(`holds ${wanted} more than once`);
+    return { ok: true, bytes: found[0] };
+}

package/dist/resolve-connector.d.ts

@@ -0,0 +1,8 @@
+export type Resolution = {
+    readonly packageName: string;
+    readonly binPath: string;
+} | {
+    readonly packageName: string;
+    readonly reason: string;
+};
+export declare function resolveConnector(name: string): Resolution;

package/dist/resolve-connector.js

@@ -0,0 +1,28 @@
+import { readFileSync } from 'node:fs';
+import { createRequire } from 'node:module';
+import { dirname, join } from 'node:path';
+export function resolveConnector(name) {
+    const packageName = `@sema-agent/connector-${name}`;
+    let manifestPath;
+    try {
+        manifestPath = createRequire(new URL('../package.json', import.meta.url)).resolve(`${packageName}/package.json`);
+    }
+    catch {
+        return { packageName, reason: 'is not installed where the launcher can resolve it' };
+    }
+    let bin;
+    try {
+        bin = JSON.parse(readFileSync(manifestPath, 'utf8')).bin;
+    }
+    catch {
+        return { packageName, reason: 'has a package.json that cannot be read' };
+    }
+    let rel;
+    if (typeof bin === 'string')
+        rel = bin;
+    else if (bin !== null && typeof bin === 'object' && !Array.isArray(bin) && Object.keys(bin).length === 1)
+        rel = Object.values(bin)[0];
+    if (typeof rel !== 'string' || rel === '')
+        return { packageName, reason: 'does not declare exactly one bin' };
+    return { packageName, binPath: join(dirname(manifestPath), rel) };
+}

package/dist/scope.js

@@ -1,3 +1,5 @@
 export function isGranted(scope, tool) {
+    if (tool === '')
+        return false;
     return scope.includes('tool:*') || scope.includes(`tool:${tool}`);
 }

package/dist/secrets.d.ts

@@ -2,6 +2,7 @@ export interface SecretHandle {
     readonly connector: string;
     readonly name: string;
     readonly source: 'file';
+    readonly secret: boolean;
     readonly expiresAt?: string;
 }
 export type SecretListing = {
@@ -13,21 +14,35 @@ export type SecretListing = {
 };
 export interface PutOptions {
     readonly expiresAt?: string;
+    readonly secret?: boolean;
 }
 export interface FileSecrets {
+    readonly directory: string;
     list(): SecretListing;
     put(name: string, value: string, options?: PutOptions): SecretHandle;
+    withLock<T>(fn: () => T): T;
 }
+export declare const isSecret: (handle: SecretHandle) => boolean;
 export declare const HANDLE_NAME: RegExp;
+export declare const LOCK_FILE = ".auth.lock";
+export declare const LOCK_WAIT_MS = 5000;
 export declare function handleDirectory(configDir: string, connector: string): string;
 export declare function handleFilePath(configDir: string, handle: SecretHandle): string;
 export declare function readSecretValue(configDir: string, handle: SecretHandle): Buffer;
 export interface LaunchSecrets {
+    readonly handles: readonly SecretHandle[];
     readonly files: ReadonlyMap<string, string>;
     readonly values: readonly string[];
     remove(): void;
 }
-export declare function copyForLaunch(configDir: string, connector: string, handles: readonly SecretHandle[]): LaunchSecrets;
+export type LaunchCopy = ({
+    readonly ok: true;
+} & LaunchSecrets) | {
+    readonly ok: false;
+    readonly problem: string;
+};
+export declare const NO_HANDLES: LaunchSecrets;
+export declare function copyForLaunch(configDir: string, connector: string): LaunchCopy;
 export declare function openFileSecrets(options: {
     readonly configDir: string;
     readonly connector: string;