@ashulab/carrier 0.0.0-stage → 0.1.0

package/dist/integrations/github.js

@@ -0,0 +1,231 @@
+import { execFile } from 'node:child_process';
+import { promisify } from 'node:util';
+import { z } from 'zod';
+import { epicSlicesQuery, epicSlicesResponse, pullRequestOf, pullRequestQuery, pullRequestResponse, sliceOf, threadsOf } from './github-graphql.js';
+import { redact } from '../logging/redact.js';
+import { AuthError, GitHubUnavailable, epicLabel, readyLabel } from '../orchestration/github.js';
+const execFileAsync = promisify(execFile);
+const PER_PAGE = 100;
+const repoPattern = /^[\w.-]+\/[\w.-]+$/;
+// Forge reports every failure with exit 1, so credentials problems are told apart by their stderr.
+const authStderr = /Agent not found:|Private key for .* not found or unreadable|App JWT was rejected|has no installations|Bad credentials|HTTP 401|not logged in|gh auth login/i;
+const networkStderr = /error connecting|dial tcp|no such host|ENOTFOUND|ECONNRESET|ECONNREFUSED|ETIMEDOUT|EAI_AGAIN|network is unreachable|i\/o timeout|connection reset|TLS handshake/i;
+/** Any non-2xx answer that is neither "GitHub is down" nor "bad credentials", e.g. 404 or 422. */
+class HttpError extends Error {
+    status;
+    constructor(status, body) {
+        super(`GitHub answered HTTP ${status}: ${body.slice(0, 300)}`);
+        this.status = status;
+    }
+}
+const issueRow = z.object({ number: z.int().positive(), title: z.string(), labels: z.array(z.object({ name: z.string() })), pull_request: z.unknown().optional() });
+// GitHub does not require `sub_issues_summary` on an issue.
+const epicRow = issueRow.extend({ sub_issues_summary: z.object({ total: z.int().nonnegative(), completed: z.int().nonnegative() }).optional() });
+const issueDetail = z.object({ state: z.enum(['open', 'closed']), labels: z.array(z.object({ name: z.string() })) });
+const pullRow = z.object({
+    number: z.int().positive(),
+    state: z.enum(['open', 'closed']),
+    merged_at: z.string().nullable(),
+    body: z.string().nullable(),
+    head: z.object({ sha: z.string().min(1), ref: z.string().min(1) }),
+});
+const labelRow = z.object({ name: z.string() });
+function retryAfterMs(headers) {
+    const retryAfter = Number(headers['retry-after']);
+    if (headers['retry-after'] !== undefined && Number.isFinite(retryAfter))
+        return retryAfter * 1000;
+    const reset = Number(headers['x-ratelimit-reset']);
+    if (headers['x-ratelimit-remaining'] === '0' && Number.isFinite(reset))
+        return Math.max(0, reset * 1000 - Date.now());
+    return undefined;
+}
+function statusFailure(profile, status, headers, body) {
+    const wait = retryAfterMs(headers);
+    const rateLimited = status === 429 || (status === 403 && (wait !== undefined || /rate limit/i.test(body)));
+    if (rateLimited)
+        return new GitHubUnavailable('rate_limited', `HTTP ${status}`, wait);
+    if (status >= 500)
+        return new GitHubUnavailable('server_error', `HTTP ${status}`, wait);
+    if (status === 401)
+        return new AuthError(profile);
+    return new HttpError(status, body);
+}
+// `gh api -i` prints the status line, the headers, a blank line and the body, also on errors.
+function parseResponse(stdout) {
+    const match = /^HTTP\/\S+ (\d{3})[^\n]*\n([\s\S]*?)\r?\n\r?\n([\s\S]*)$/.exec(stdout);
+    if (!match)
+        return null;
+    const headers = {};
+    for (const line of match[2].split(/\r?\n/)) {
+        const colon = line.indexOf(':');
+        if (colon > 0)
+            headers[line.slice(0, colon).toLowerCase()] = line.slice(colon + 1).trim();
+    }
+    return { status: Number(match[1]), headers, body: match[3] };
+}
+/** The REST bucket's numbers from the `x-ratelimit-*` headers; null when any is missing or malformed, and for the GraphQL bucket, which is separate. */
+function rateLimitOf(headers) {
+    const resource = headers['x-ratelimit-resource'];
+    if (resource !== undefined && resource !== 'core')
+        return null;
+    const [limit, remaining, reset] = ['limit', 'remaining', 'reset'].map((name) => (/^\d+$/.test(headers[`x-ratelimit-${name}`] ?? '') ? Number(headers[`x-ratelimit-${name}`]) : NaN));
+    if (!Number.isSafeInteger(limit) || !Number.isSafeInteger(remaining) || !Number.isSafeInteger(reset))
+        return null;
+    return { remaining: remaining, limit: limit, resetAt: reset * 1000 };
+}
+export class GhGitHub {
+    options;
+    latestRateLimit = null;
+    constructor(options) {
+        this.options = options;
+    }
+    rateLimit() {
+        return this.latestRateLimit;
+    }
+    /** Keeps the numbers of the latest answer, a failure included: the call that hit the limit says it is 0. */
+    track(headers) {
+        this.latestRateLimit = rateLimitOf(headers) ?? this.latestRateLimit;
+    }
+    /** One `gh api` call: argv separated, no shell. Returns the body of a 2xx answer, throws for anything else. */
+    async api(repo, args) {
+        if (!repoPattern.test(repo))
+            throw new Error(`invalid repo "${repo}"`);
+        const { profile, executable } = this.options;
+        const profileName = profile.kind === 'forge' ? profile.agent : 'gh';
+        const [command, ...prefix] = profile.kind === 'forge' ? [executable ?? 'agent-forge', 'gh', profile.agent] : [executable ?? 'gh'];
+        try {
+            const { stdout } = await execFileAsync(command, [...prefix, 'api', '-i', ...args], {
+                cwd: this.options.cwd(repo),
+                timeout: this.options.timeoutMs,
+                maxBuffer: 64 * 1024 * 1024,
+                encoding: 'utf8',
+            });
+            const response = parseResponse(stdout);
+            if (!response)
+                throw new Error(`unexpected gh output: ${stdout.slice(0, 200)}`);
+            this.track(response.headers);
+            if (response.status >= 300)
+                throw statusFailure(profileName, response.status, response.headers, response.body);
+            return response.body;
+        }
+        catch (error) {
+            if (error instanceof GitHubUnavailable || error instanceof AuthError || error instanceof HttpError)
+                throw error;
+            const failed = error;
+            // Exceeding maxBuffer also kills the child, but that is not a timeout.
+            if (failed.killed && failed.code !== 'ERR_CHILD_PROCESS_STDIO_MAXBUFFER')
+                throw new GitHubUnavailable('timeout', `timed out after ${this.options.timeoutMs} ms`);
+            const response = parseResponse(failed.stdout ?? '');
+            if (response) {
+                this.track(response.headers);
+                throw statusFailure(profileName, response.status, response.headers, response.body);
+            }
+            const stderr = failed.stderr ?? '';
+            if (authStderr.test(stderr))
+                throw new AuthError(profileName);
+            if (networkStderr.test(stderr))
+                throw new GitHubUnavailable('network', redact(stderr).replace(/\s+/g, ' ').trim().slice(0, 300));
+            throw new Error(`gh failed: ${(stderr || failed.message).trim().slice(0, 300)}`);
+        }
+    }
+    async get(repo, path, schema) {
+        return schema.parse(JSON.parse(await this.api(repo, [`repos/${repo}/${path}`])));
+    }
+    /** Pages of a list endpoint, fetched lazily until one comes back short. */
+    async *pages(repo, path, params, row) {
+        for (let page = 1;; page++) {
+            const query = new URLSearchParams({ ...params, per_page: String(PER_PAGE), page: String(page) });
+            const rows = await this.get(repo, `${path}?${query}`, z.array(row));
+            yield rows;
+            if (rows.length < PER_PAGE)
+                return;
+        }
+    }
+    async listReadyEpics(repo) {
+        const epics = [];
+        for await (const rows of this.pages(repo, 'issues', { labels: readyLabel, state: 'open' }, issueRow)) {
+            for (const row of rows)
+                if (row.pull_request === undefined)
+                    epics.push({ issue: row.number, title: row.title, labels: row.labels.map((label) => label.name) });
+        }
+        return epics;
+    }
+    async listOpenEpics(repo) {
+        const epics = [];
+        for await (const rows of this.pages(repo, 'issues', { labels: epicLabel, state: 'open' }, epicRow)) {
+            for (const row of rows) {
+                if (row.pull_request === undefined)
+                    epics.push({ issue: row.number, title: row.title, labels: row.labels.map((label) => label.name), slices: row.sub_issues_summary ? { done: row.sub_issues_summary.completed, total: row.sub_issues_summary.total } : null });
+            }
+        }
+        return epics;
+    }
+    async readSlices(repo, issue) {
+        const [owner = '', name = ''] = repo.split('/');
+        const body = await this.api(repo, ['graphql', '-f', `query=${epicSlicesQuery}`, '-f', `owner=${owner}`, '-f', `name=${name}`, '-F', `number=${issue}`]);
+        const { subIssues } = epicSlicesResponse.parse(JSON.parse(body)).data.repository.issue;
+        return { slices: subIssues.nodes.map(sliceOf), truncated: subIssues.pageInfo.hasNextPage };
+    }
+    async readPullRequest(repo, number) {
+        const [owner = '', name = ''] = repo.split('/');
+        const body = await this.api(repo, ['graphql', '-f', `query=${pullRequestQuery}`, '-f', `owner=${owner}`, '-f', `name=${name}`, '-F', `number=${number}`]);
+        const response = pullRequestResponse.parse(JSON.parse(body));
+        return { pr: pullRequestOf(response), ...threadsOf(response) };
+    }
+    async getIssue(repo, issue) {
+        const { state, labels } = await this.get(repo, `issues/${issue}`, issueDetail);
+        return { state, labels: labels.map((label) => label.name) };
+    }
+    async getPullRequest(repo, number) {
+        try {
+            const pull = await this.get(repo, `pulls/${number}`, pullRow);
+            return { state: pull.merged_at === null ? pull.state : 'merged', body: pull.body ?? '', headSha: pull.head.sha, headRef: pull.head.ref };
+        }
+        catch (error) {
+            if (error instanceof HttpError && error.status === 404)
+                return null;
+            throw error;
+        }
+    }
+    async findPullRequest(repo, issue) {
+        // Closing keywords GitHub honors, optionally with an explicit reference to this same repo.
+        const closes = new RegExp(`\\b(?:close[sd]?|fix(?:e[sd])?|resolve[sd]?):?\\s+(?:${repo.replaceAll('.', '\\.')})?#${issue}(?!\\d)`, 'i');
+        // Newest first: the epic's final PR is nearly always on the first page.
+        for await (const rows of this.pages(repo, 'pulls', { state: 'all', sort: 'created', direction: 'desc' }, pullRow)) {
+            const found = rows.find((pull) => closes.test(pull.body ?? ''));
+            if (found)
+                return { number: found.number, state: found.merged_at === null ? found.state : 'merged', headSha: found.head.sha };
+        }
+        return null;
+    }
+    async setLabels(repo, issue, { add, remove }) {
+        // Add first so the issue is never left without its state label.
+        if (add.length > 0)
+            await this.api(repo, ['-X', 'POST', `repos/${repo}/issues/${issue}/labels`, ...add.flatMap((label) => ['-f', `labels[]=${label}`])]);
+        for (const label of remove) {
+            try {
+                await this.api(repo, ['-X', 'DELETE', `repos/${repo}/issues/${issue}/labels/${encodeURIComponent(label)}`]);
+            }
+            catch (error) {
+                // Removing a label the issue lacks is a no-op.
+                if (!(error instanceof HttpError && error.status === 404 && /label does not exist/i.test(error.message)))
+                    throw error;
+            }
+        }
+    }
+    async listLabels(repo) {
+        const names = [];
+        for await (const rows of this.pages(repo, 'labels', {}, labelRow))
+            names.push(...rows.map((row) => row.name));
+        return names;
+    }
+    async createLabel(repo, name, { color, description }) {
+        try {
+            await this.api(repo, ['-X', 'POST', `repos/${repo}/labels`, '-f', `name=${name}`, '-f', `color=${color}`, '-f', `description=${description}`]);
+        }
+        catch (error) {
+            if (!(error instanceof HttpError && error.status === 422 && /already_exists/.test(error.message)))
+                throw error;
+        }
+    }
+}

package/dist/logging/logger.js

@@ -0,0 +1,53 @@
+import { appendFileSync, mkdirSync, openSync, writeSync } from 'node:fs';
+import { dirname, join } from 'node:path';
+import { format } from 'node:util';
+import { configureUniversalLogger, LogLevel } from '@ashulab/universal-logger';
+import { redact } from './redact.js';
+/**
+ * Sends everything the process writes with console.* (the library and its dependencies included) to `file`,
+ * so the panel is the only thing that writes to the terminal. The level is not in the library's JSON: the
+ * console method used adds it. writeSync, not a stream: when it returns the line is in the kernel, so SIGINT loses nothing.
+ */
+export function installFileLogging(file, minLevel) {
+    process.env.NODE_ENV ??= 'production'; // 'development' makes the library emit several lines per log
+    mkdirSync(dirname(file), { recursive: true });
+    const fd = openSync(file, 'a');
+    for (const method of ['log', 'info', 'warn', 'error', 'debug']) {
+        console[method] = (...args) => void writeSync(fd, redact(`[${method}] ${format(...args)}\n`));
+    }
+    configureUniversalLogger({ enabled: true, minLevel });
+}
+/** Context every log of an execution carries when it applies. */
+export const taskContext = (c) => ({
+    repo: c.repo,
+    epic: c.epic,
+    executionId: c.executionId,
+    role: c.stage === 'review' ? 'reviewer' : 'author',
+    stage: c.stage,
+    ...(c.round && { round: c.round }),
+});
+const MAX_PENDING = 64 * 1024;
+/**
+ * Where the whole output of one execution goes: `<logs>/executions/<executionId>/{stdout,stderr}.log`.
+ * Only whole lines are redacted and written, so a token split across two chunks is still caught; `flush` writes what is left when the run ends.
+ */
+export function executionOutput(logs, executionId) {
+    const dir = join(logs, 'executions', executionId);
+    mkdirSync(dir, { recursive: true });
+    const pending = { stdout: '', stderr: '' };
+    const append = (stream, text) => text && appendFileSync(join(dir, `${stream}.log`), redact(text));
+    return {
+        write(stream, chunk) {
+            const text = pending[stream] + chunk;
+            // a line that never ends is written anyway past the cap: memory over a perfect redaction
+            const cut = text.length > MAX_PENDING ? text.length : text.lastIndexOf('\n') + 1;
+            append(stream, text.slice(0, cut));
+            pending[stream] = text.slice(cut);
+        },
+        flush() {
+            append('stdout', pending.stdout);
+            append('stderr', pending.stderr);
+            pending.stdout = pending.stderr = '';
+        },
+    };
+}

package/dist/logging/redact.js

@@ -0,0 +1,21 @@
+// Replaces what looks like a secret. A pattern list can miss one (ADR 0006): this is a mitigation, not a guarantee.
+// Order matters: a private key block goes first, because its body would match nothing else but is the biggest leak.
+const patterns = [
+    // To the END line, or to the end of the text when the block was cut.
+    /-----BEGIN [A-Z ]*PRIVATE KEY(?: BLOCK)?-----[\s\S]*?(?:-----END [A-Z ]*PRIVATE KEY(?: BLOCK)?-----|$)/g,
+    // The header's value, with or without a scheme (Bearer, Basic, token).
+    /\bAuthorization\s*:\s*(?:(?:Bearer|Basic|Digest|Negotiate|Token)\s+)?[^\s"']+/gi,
+    // NAME=value where the name holds SECRET, TOKEN, PASSWORD or KEY; the value is quoted or runs to a space.
+    /\b\w*(?:SECRET|TOKEN|PASSWORD|KEY)\w*=(?!=)(?:"[^"]*"|'[^']*'|[^\s"']+)/gi,
+    /\b(?:gh[pousr]_|github_pat_)[A-Za-z0-9_]+/g, // GitHub: classic, OAuth, user, server, refresh, fine-grained
+    /\bsk-[A-Za-z0-9_-]{16,}/g, // OpenAI, Anthropic (sk-ant-), OpenRouter
+    /\b[sr]k_(?:live|test)_[A-Za-z0-9]{16,}/g, // Stripe
+    /\bAKIA[0-9A-Z]{16}\b/g, // AWS access key id
+    /\bAIza[0-9A-Za-z_-]{35}/g, // Google API key
+    /\bxox[abeprs]-[A-Za-z0-9-]{10,}/g, // Slack
+    /\bglpat-[A-Za-z0-9_-]{20,}/g, // GitLab
+    /\bnpm_[A-Za-z0-9]{36}\b/g, // npm
+    /\bhf_[A-Za-z0-9]{30,}/g, // Hugging Face
+    /\beyJ[\w-]{10,}\.[\w-]{10,}\.[\w-]{10,}/g, // JWT
+];
+export const redact = (text) => patterns.reduce((out, pattern) => out.replace(pattern, '[redacted]'), text);

package/dist/orchestration/blockers.js

@@ -0,0 +1,25 @@
+import { and, desc, eq, sql } from 'drizzle-orm';
+import { callerResults } from '../storage/caller-results.js';
+import { executions } from '../storage/executions.js';
+/** The blockers of the epic's latest review that asked for changes: what the fix turn has to answer. */
+export function latestBlockers(db, epicId) {
+    const last = db
+        .select({ result: callerResults.result })
+        .from(callerResults)
+        .innerJoin(executions, eq(executions.executionId, callerResults.executionId))
+        .where(and(eq(executions.epicId, epicId), eq(executions.stage, 'review'), eq(callerResults.outcome, 'changes_requested')))
+        .orderBy(desc(sql `executions.rowid`))
+        .get();
+    return last ? JSON.parse(last.result).blockers : [];
+}
+/** The observations of the epic's latest review when it completed: a fix right after it is the observations pass, null otherwise (that fix is for blockers). */
+export function latestObservations(db, epicId) {
+    const last = db
+        .select({ outcome: callerResults.outcome, result: callerResults.result })
+        .from(callerResults)
+        .innerJoin(executions, eq(executions.executionId, callerResults.executionId))
+        .where(and(eq(executions.epicId, epicId), eq(executions.stage, 'review')))
+        .orderBy(desc(sql `executions.rowid`))
+        .get();
+    return last?.outcome === 'completed' ? (JSON.parse(last.result).observations ?? []) : null;
+}

package/dist/orchestration/changes-slices.js

@@ -0,0 +1,5 @@
+/**
+ * Whether a shell command of the agent is one of the `gh` commands that change an epic's slices (take, open, merge, close).
+ * Only a signal to look: the data still comes from GitHub, so a missed or false match costs one query or one timer period.
+ */
+export const changesSlices = (command) => /(^|[\s;&|(])gh\s+(?:issue\s+(?:edit|close)|pr\s+(?:create|merge|ready|close))(?=\s|$)/.test(command);

package/dist/orchestration/clock.js

@@ -0,0 +1,7 @@
+export const realClock = {
+    now: () => Date.now(),
+    after(ms, fn) {
+        const timer = setTimeout(fn, ms);
+        return () => clearTimeout(timer);
+    },
+};