copilot-operator 0.1.17 → 0.1.19

package/dist/src/vcs/snapshot.js

@@ -22,12 +22,15 @@
  * Like the rest of `vcs/`: nothing is reset, stashed, deleted or rewritten. The snapshot adds a
  * branch and a commit.
  */
-import { mkdir, readFile, rm, writeFile } from 'node:fs/promises';
+import { mkdir, readFile, rm, stat, writeFile } from 'node:fs/promises';
 import { tmpdir } from 'node:os';
 import { dirname, isAbsolute, join, relative, resolve } from 'node:path';
 import { looksGenerated } from './commitHygiene.js';
 import { branchNameFrom, freeBranchName, git, gitBytes, isValidBranchName, RUNNER_EMAIL } from './git.js';
-import { repoDirOf, sessionStart } from './taskVcs.js';
+import { repoDirOf, sessionBranchName, sessionStart } from './taskVcs.js';
+import { inputChanges, inputFilesInTree, inputSettings, inputSumsAt, missingInputPatterns, recaptureInputs } from './inputs.js';
+import { artifactPatterns } from './artifacts.js';
+import { inScope } from './scope.js';
 /** The session's dirty-tree policy, absent fields filled in: reject, and asking first. */
 export function dirtyPolicy(vcs) {
     const policy = vcs?.dirtyWorktree?.policy;
@@ -128,13 +131,18 @@ async function startProblem(session, dir, branch, head, allSessions) {
  */
 export async function planSnapshot(session, allSessions = async () => []) {
     const { policy, requireApproval } = dirtyPolicy(session.vcs);
+    const inputs = inputSettings(session.vcs);
     const repoDir = repoDirOf(session);
     const plan = { needed: false, ok: false, policy, requireApproval, repoDir, branch: null, head: null, entries: [] };
-    if (!session.vcs?.enabled || policy === 'reject' || session.vcsBaseCommit)
+    if (!session.vcs?.enabled || (policy === 'reject' && !inputs))
+        return plan;
+    if (session.vcsBaseCommit && (!inputs || session.vcs.startFrom === 'existing-branch'))
         return plan;
     const top = repoDir ? await topOf(repoDir) : null;
     if (!top)
         return { ...plan, problem: `${repoDir || 'The repository folder'} is not a git repository.` };
+    if (session.vcsBaseCommit)
+        return await planRecapture(session, top, plan);
     const status = await statusEntries(top);
     if (!status)
         return { ...plan, problem: 'git status could not be read.' };
@@ -142,13 +150,43 @@ export async function planSnapshot(session, allSessions = async () => []) {
     const headR = await git(top, ['rev-parse', '--verify', '--quiet', 'HEAD']);
     plan.branch = branchR.ok && branchR.stdout !== 'HEAD' ? branchR.stdout : null;
     plan.head = headR.ok && headR.stdout ? headR.stdout : null;
-    if (status.length === 0)
-        return plan;
-    plan.needed = true;
     // The project folder, when it is a folder inside the repository: a change outside it is not this session's.
     const project = session.projectDir?.trim() ? relative(top, resolve(session.projectDir.trim())).replace(/\\/g, '/') : '';
     const inProject = (p) => !project || project.startsWith('..') || isAbsolute(project) || p.toLowerCase() === project.toLowerCase() || p.toLowerCase().startsWith(`${project.toLowerCase()}/`);
-    for (const s of status) {
+    const size = async (p) => (await stat(join(top, p)).catch(() => null))?.size;
+    /*
+     * The operator's input files that are not committed as they are: new, changed, or ignored. They
+     * are taken whatever the dirty-tree policy, or the snapshot is not; the rest of the tree is the
+     * policy's. A secrets file or one outside the project is never an input.
+     */
+    const inputPaths = new Set();
+    if (inputs) {
+        const dirtyNow = new Set(status.map((s) => s.path));
+        for (const f of await inputFilesInTree(top, inputs.patterns)) {
+            if (f.kind === 'tracked' && !dirtyNow.has(f.path))
+                continue;
+            inputPaths.add(f.path);
+            const blocked = looksGenerated(f.path)?.reason === 'a secrets file' ? 'a secrets file is never taken as an input' : !inProject(f.path) ? `outside the project folder ${project}` : undefined;
+            plan.entries.push({ path: f.path, kind: f.kind, input: true, size: await size(f.path), allowed: blocked ? [] : ['include'], choice: blocked ? null : 'include', reason: blocked ?? 'an input file: committed with the starting snapshot, then read-only' });
+        }
+    }
+    const others = status.filter((s) => !inputPaths.has(s.path));
+    if (policy === 'reject') {
+        if (plan.entries.length === 0)
+            return plan;
+        plan.needed = true;
+        if (others.length > 0) {
+            plan.problem =
+                `besides the input files there are other uncommitted changes (${someOf(others.map((o) => o.path))}). ` +
+                    'Commit them yourself, or choose "Take them as a starting snapshot" under "Uncommitted changes".';
+            return plan;
+        }
+    }
+    if (plan.entries.length === 0 && others.length === 0)
+        return plan;
+    plan.needed = true;
+    plan.requireApproval = (others.length > 0 && requireApproval) || (plan.entries.some((e) => e.input) && !!inputs?.requireApproval);
+    for (const s of policy === 'reject' ? [] : others) {
         const generated = looksGenerated(s.path);
         const outside = !inProject(s.path);
         if (!s.untracked) {
@@ -157,21 +195,31 @@ export async function planSnapshot(session, allSessions = async () => []) {
                 : outside
                     ? `outside the project folder ${project}; it is tracked, so it cannot be left out. Commit or undo that change yourself`
                     : undefined;
-            plan.entries.push({ path: s.path, ...(s.from ? { from: s.from } : {}), kind: 'tracked', allowed: blocked ? [] : ['include'], choice: blocked ? null : 'include', ...(blocked ? { reason: blocked } : {}) });
+            plan.entries.push({ path: s.path, ...(s.from ? { from: s.from } : {}), kind: 'tracked', size: await size(s.path), allowed: blocked ? [] : ['include'], choice: blocked ? null : 'include', ...(blocked ? { reason: blocked } : {}) });
             continue;
         }
-        const why = generated ? generated.reason : outside ? `outside the project folder ${project}` : policy === 'tracked-only-snapshot' ? 'a new file, and this session takes tracked changes only' : undefined;
-        plan.entries.push({ path: s.path, kind: 'untracked', allowed: why ? ['leave-out'] : ['include', 'leave-out'], choice: why ? 'leave-out' : 'include', ...(why ? { reason: why } : {}) });
+        const artifact = artifactPatterns(session.vcs).length > 0 && inScope(s.path, artifactPatterns(session.vcs));
+        const why = artifact
+            ? 'an artifact: kept with the run, never committed'
+            : generated ? generated.reason : outside ? `outside the project folder ${project}` : policy === 'tracked-only-snapshot' ? 'a new file, and this session takes tracked changes only' : undefined;
+        plan.entries.push({ path: s.path, kind: 'untracked', size: await size(s.path), allowed: why ? ['leave-out'] : ['include', 'leave-out'], choice: why ? 'leave-out' : 'include', ...(why ? { reason: why } : {}) });
     }
     if (policy === 'snapshot') {
         for (const path of await scopedIgnored(top, scopedPatterns(session))) {
-            if (looksGenerated(path) || !inProject(path))
+            if (looksGenerated(path) || !inProject(path) || inputPaths.has(path))
                 continue;
             plan.entries.push({ path, kind: 'ignored', allowed: ['include', 'leave-out'], choice: 'leave-out', reason: "ignored by git, and a task's scope names it: taken only when you tick it" });
         }
     }
     const blocked = plan.entries.filter((e) => e.allowed.length === 0);
-    if (!plan.head)
+    // A pattern that matches nothing anywhere is said now, before anything is committed for approval.
+    const missing = inputs ? await missingInputPatterns(session, repoDir, top, allSessions) : [];
+    if (missing.length > 0) {
+        plan.problem =
+            `the input file pattern(s) ${missing.map((m) => `"${m}"`).join(', ')} match no file in the project, and nothing earlier captured them. ` +
+                'Correct "Input files" (a "*" stays inside one folder; "**" crosses folders), or put the files in the project.';
+    }
+    else if (!plan.head)
         plan.problem = 'the repository has no commit yet, so there is nothing for a snapshot to sit on. Make the first commit yourself.';
     else if (blocked.length > 0)
         plan.problem = `${blocked.length} change(s) can be neither taken nor left out: ${blocked.map((e) => `${e.path} (${e.reason})`).join('; ')}.`;
@@ -187,6 +235,62 @@ export async function planSnapshot(session, allSessions = async () => []) {
     plan.ok = !plan.problem;
     return plan;
 }
+/**
+ * After the session's start: the input files changed or added since it was recorded, to be taken
+ * onto its line of work (see `recaptureInputs`). Anything else uncommitted is in the way, as it is
+ * before a task, and refuses.
+ */
+async function planRecapture(session, top, plan) {
+    const inputs = inputSettings(session.vcs);
+    if (!inputs || !session.vcsBaseCommit)
+        return plan;
+    const branchR = await git(top, ['rev-parse', '--abbrev-ref', 'HEAD']);
+    const headR = await git(top, ['rev-parse', '--verify', '--quiet', 'HEAD']);
+    plan.branch = branchR.ok && branchR.stdout !== 'HEAD' ? branchR.stdout : null;
+    plan.head = headR.ok && headR.stdout ? headR.stdout : null;
+    const changes = await inputChanges(top, inputs.patterns, session.vcsStart?.inputs?.files);
+    if (changes.length === 0)
+        return plan;
+    plan.needed = true;
+    plan.requireApproval = inputs.requireApproval;
+    const project = session.projectDir?.trim() ? relative(top, resolve(session.projectDir.trim())).replace(/\\/g, '/') : '';
+    const inProject = (p) => !project || project.startsWith('..') || isAbsolute(project) || p.toLowerCase() === project.toLowerCase() || p.toLowerCase().startsWith(`${project.toLowerCase()}/`);
+    for (const c of changes) {
+        const blocked = looksGenerated(c.path)?.reason === 'a secrets file' ? 'a secrets file is never taken as an input' : !inProject(c.path) ? `outside the project folder ${project}` : undefined;
+        plan.entries.push({
+            path: c.path,
+            kind: c.kind,
+            input: true,
+            size: (await stat(join(top, c.path)).catch(() => null))?.size,
+            allowed: blocked ? [] : ['include'],
+            choice: blocked ? null : 'include',
+            reason: blocked ?? 'an input file changed or added after the session started: committed on its line of work, then read-only',
+        });
+    }
+    const changed = new Set(changes.map((c) => c.path));
+    const others = ((await statusEntries(top)) ?? []).filter((s) => !changed.has(s.path)).map((s) => s.path);
+    const blocked = plan.entries.filter((e) => e.allowed.length === 0);
+    if (others.length > 0) {
+        plan.problem = `besides the input files there are other uncommitted changes (${someOf(others)}). Commit or stash them yourself first.`;
+    }
+    else if (blocked.length > 0) {
+        plan.problem = `${blocked.length} file(s) cannot be taken as inputs: ${blocked.map((e) => `${e.path} (${e.reason})`).join('; ')}.`;
+    }
+    // Where they go: on the session's one branch when it has it, else on the commit tasks are cut from.
+    const sessionBranch = sessionBranchName(session);
+    const tip = session.vcs?.branchMode === 'per-session' ? (await git(top, ['rev-parse', '--verify', '--quiet', `refs/heads/${sessionBranch}^{commit}`])).stdout : '';
+    if (tip) {
+        plan.recapture = { parent: tip, ffBranch: sessionBranch };
+        plan.baselineBranch = sessionBranch;
+    }
+    else {
+        const newBranch = await freeBranchName(top, `${branchNameFrom(['input'], session.vcs?.branchPrefix || 'cop/')}/${session.id}`);
+        plan.recapture = { parent: session.vcsBaseCommit, newBranch };
+        plan.baselineBranch = newBranch;
+    }
+    plan.ok = !plan.problem;
+    return plan;
+}
 /** A path as a pattern for `.git/info/exclude` that matches that one file and nothing else. */
 function excludePattern(path) {
     return `/${path.replace(/[\\*?[]/g, (c) => `\\${c}`).replace(/ $/, '\\ ')}`;
@@ -215,16 +319,20 @@ async function leaveOut(top, session, paths) {
     return undefined;
 }
 /** The commit message: what was taken, what was left out, from where, and on whose word. */
-function snapshotMessage(session, plan, included, leftOut, approved) {
+function snapshotMessage(session, plan, taken, leftOut, approved) {
     const list = (paths) => [...paths.slice(0, 200).map((p) => `- ${p}`), ...(paths.length > 200 ? [`- and ${paths.length - 200} more`] : [])].join('\n');
+    const inputs = taken.filter((e) => e.input);
+    const rest = taken.filter((e) => !e.input).map((e) => e.path);
+    const onlyInputs = rest.length === 0;
     return [
-        'Capture operator baseline before run',
+        onlyInputs ? 'Capture user-provided inputs' : 'Capture operator baseline before run',
         '',
-        `The uncommitted changes in the repository when session "${session.name}" was first run, committed so that`,
+        `The ${onlyInputs ? "operator's input files" : 'uncommitted changes in the repository'} when session "${session.name}" was first run, committed so that`,
         "the session starts from them and every task's commit shows only that task's work.",
-        '',
-        `Taken (${included.length}):`,
-        list(included),
+        ...(inputs.length > 0
+            ? ['', `Input files, read by the work and not changed by it (${inputs.length}):`, list(inputs.map((e) => `${e.path}${e.size !== undefined ? ` (${e.size} bytes, ${e.kind})` : ''}`))]
+            : []),
+        ...(rest.length > 0 ? ['', `Taken (${rest.length}):`, list(rest)] : []),
         ...(leftOut.length > 0 ? ['', `Left out, in .git/info/exclude (${leftOut.length}):`, list(leftOut)] : []),
         '',
         `Taken from ${plan.branch ?? 'a detached HEAD'} at ${(plan.head ?? '').slice(0, 8)}, ${approved ? "on the operator's approval of this list" : 'automatically (requireApproval: false)'}.`,
@@ -240,11 +348,55 @@ function snapshotMessage(session, plan, included, leftOut, approved) {
  * `approved`, every file takes its default and anything to be left out refuses (see `DirtyWorktree`).
  */
 export async function takeSnapshot(session, opts, bus, saveSession, allSessions = async () => []) {
-    if (dirtyPolicy(session.vcs).policy === 'reject') {
+    if (dirtyPolicy(session.vcs).policy === 'reject' && !inputSettings(session.vcs)) {
         return { ok: false, problem: 'this session does not take uncommitted changes as a snapshot. Choose that under Version control → "Uncommitted changes" first.' };
     }
-    if (session.vcsBaseCommit)
-        return { ok: false, problem: 'this session has already started; a snapshot is taken only before its first task.' };
+    if (session.vcsBaseCommit) {
+        // Started already: only input files changed or added since can be taken, onto its line of work.
+        const plan = await planSnapshot(session, allSessions);
+        if (!plan.needed || !plan.recapture) {
+            return { ok: false, problem: 'this session has already started, and no input file has changed since: a snapshot is taken only before its first task.' };
+        }
+        if (!plan.ok)
+            return { ok: false, problem: plan.problem ?? 'the input files cannot be taken.' };
+        const paths = plan.entries.map((e) => e.path);
+        if (opts.approved) {
+            const shown = Object.keys(opts.choices ?? {}).sort();
+            if (JSON.stringify(shown) !== JSON.stringify([...paths].sort()) || paths.some((p) => opts.choices?.[p] !== 'include')) {
+                return { ok: false, problem: 'the input files changed since the list was shown. Look at the list again; nothing was done.' };
+            }
+        }
+        const made = await recaptureInputs(session, plan.repoDir, paths, plan.recapture);
+        if ('problem' in made)
+            return { ok: false, problem: made.problem };
+        const settings = inputSettings(session.vcs);
+        const top = (await topOf(plan.repoDir));
+        const before = session.vcsStart;
+        const start = {
+            ...before,
+            inputs: {
+                patterns: settings?.patterns ?? [],
+                files: await inputSumsAt(top, made.commit, settings?.patterns ?? []),
+                readOnly: settings?.readOnly ?? true,
+                ...(before.inputs?.carried ? { carried: before.inputs.carried } : {}),
+                recaptured: [...(before.inputs?.recaptured ?? []), { commit: made.commit, branch: made.branch, paths: paths.slice(0, 200), approved: opts.approved }],
+            },
+        };
+        await saveSession((s) => {
+            s.vcsStart = start;
+            // A commit tasks are cut from moves to the inputs' commit; a session branch moved forward itself.
+            if (plan.recapture?.newBranch)
+                s.vcsBaseCommit = made.commit;
+        });
+        bus.publish({
+            sessionId: session.id,
+            type: 'vcs-inputs-recaptured',
+            level: 'info',
+            message: `${paths.length} input file(s) changed or added since the session started are now the commit ${made.commit.slice(0, 8)} on ${made.branch}; the next task starts from it`,
+            data: { commit: made.commit, branch: made.branch, paths: paths.slice(0, 50) },
+        });
+        return { ok: true, start };
+    }
     const plan = await planSnapshot(session, allSessions);
     if (!plan.needed)
         return { ok: false, problem: 'there is nothing uncommitted to take.' };
@@ -320,7 +472,7 @@ export async function takeSnapshot(session, opts, bus, saveSession, allSessions
     finally {
         await rm(list, { force: true });
     }
-    const message = snapshotMessage(session, plan, take.map((e) => e.path), leftOutAll, opts.approved);
+    const message = snapshotMessage(session, plan, take, leftOutAll, opts.approved);
     const commit = await git(top, ['-c', 'user.name=copilot-operator', '-c', `user.email=${RUNNER_EMAIL}`, 'commit', '--no-verify', '-q', '-m', message]);
     if (!commit.ok)
         return await back(`the snapshot could not be committed: ${commit.stderr || commit.stdout}.`);