@bevel-software/platform-core-backend 0.32.0 → 0.34.0

package/src/modules/workspace/__tests__/agent-uploads.test.ts

@@ -18,6 +18,7 @@ import { WorkflowHooks } from '../../workflow/workflow-hooks.js';
 import { ToolDescriptionNotes } from '../agent-access.gate.js';
 import { SpillStore } from '../spill-store.js';
 import { DocExtractService } from '../file-readers/doc-extract.service.js';
+import { WorkerDocumentExtractor } from '../file-readers/extract-in-worker.js';
 import type { IAccessControl } from '../../access/access-control.interface.js';
 import { normalizeWorkspacePath } from '../../kb-fs/repo-path.js';
 import {
@@ -243,7 +244,7 @@ async function start(options: StartOptions = {}): Promise<string> {
     fakeAuth,
     toolHandler,
     new SpillStore(join(tmpdir(), 'bevel-test-spills')),
-    new DocExtractService(docCacheDir),
+    new DocExtractService(docCacheDir, new WorkerDocumentExtractor()),
     access,
     testKbContext({ kbDirName: KB_DIR }),
     { recoveryBotEmail: 'recovery-bot@bevel.local', hooks: new WorkflowHooks(), notes: new ToolDescriptionNotes() },
@@ -535,6 +536,83 @@ describe('the upload route', () => {
     expect(await readdir(uploadsDir)).toEqual([]);
   });
 
+  it('discards a token on request, with the bytes sent against it, the token in a header', async () => {
+    const base = await start();
+    const discard = (token: string) =>
+      fetch(`${base}/api/agent/uploads`, { method: 'DELETE', headers: { 'x-upload-token': token } });
+    const { token } = await request(base);
+    expect((await send(base, token, 'a.md', Buffer.from('a'))).status).toBe(200);
+    expect(await readdir(uploadsDir)).toHaveLength(1);
+    expect((await discard(token)).status).toBe(204);
+    expect(await readdir(uploadsDir)).toEqual([]);
+    // Spent: neither an apply nor a second discard can use it.
+    const applied = await call(base, 'apply_file_upload', { branch: DRAFT, token, destination: KB_DIR });
+    expect(applied.ok).toBe(false);
+    expect(batches).toEqual([]);
+    const again = await discard(token);
+    expect(again.status).toBe(404);
+    expect((await json<{ error: string }>(again)).error).toBe(UPLOAD_TOKEN_REFUSAL);
+    // A token nothing was sent against goes the same way, and its slot with it.
+    const { token: unsent } = await request(base);
+    expect((await discard(unsent)).status).toBe(204);
+    expect((await send(base, unsent, 'b.md', Buffer.from('b'))).status).toBe(404);
+  });
+
+  it('refuses to discard a token an apply is holding, and leaves its bytes', async () => {
+    const base = await start();
+    const { token } = await request(base);
+    await send(base, token, 'a.md', Buffer.from('a'));
+    uploads.claim(token, 'u');
+    const res = await fetch(`${base}/api/agent/uploads`, { method: 'DELETE', headers: { 'x-upload-token': token } });
+    expect(res.status).toBe(404);
+    expect(await readdir(uploadsDir)).toHaveLength(1);
+    uploads.release(token);
+  });
+
+  it('refuses to discard a token whose bytes are still arriving, and the upload finishes whole', async () => {
+    const base = await start();
+    const { token } = await request(base);
+    const head = Buffer.from('the first half, ');
+    const tail = Buffer.from('and the second half');
+    let finish!: () => void;
+    const body = new ReadableStream<Uint8Array>({
+      async start(controller) {
+        controller.enqueue(new Uint8Array(head));
+        await new Promise<void>((resolve) => {
+          finish = resolve;
+        });
+        controller.enqueue(new Uint8Array(tail));
+        controller.close();
+      },
+    });
+    const arriving = fetch(`${base}/api/agent/uploads?filename=held.md`, {
+      method: 'POST',
+      headers: { 'content-type': 'application/octet-stream', 'x-upload-token': token },
+      body,
+      // Node's fetch requires this for a streamed request body.
+      duplex: 'half',
+    } as RequestInit & { duplex: 'half' });
+    // Waited for, not assumed from a delay: the body has reached the store
+    // and its first bytes are staged, with the request still open.
+    const staged = Date.now() + 5_000;
+    while ((await readdir(uploadsDir).catch(() => [])).length === 0) {
+      expect(Date.now(), 'the upload never reached the store').toBeLessThan(staged);
+      await new Promise((r) => setTimeout(r, 10));
+    }
+
+    const res = await fetch(`${base}/api/agent/uploads`, { method: 'DELETE', headers: { 'x-upload-token': token } });
+    expect(res.status).toBe(404);
+    expect((await json<{ error: string }>(res)).error).toBe(UPLOAD_TOKEN_REFUSAL);
+    // The file being written is still there, and the token still good.
+    expect(await readdir(uploadsDir)).toHaveLength(1);
+
+    finish();
+    expect((await arriving).status).toBe(200);
+    const whole = Buffer.concat([head, tail]);
+    expect((await apply(base, { branch: DRAFT, token, destination: `${KB_DIR}/Held` })).count).toBe(1);
+    expect(sha(await readFile(join(tempDir, KB_DIR, 'Held/held.md')))).toBe(sha(whole));
+  });
+
   it('refuses a second upload against the same token', async () => {
     const base = await start();
     const { token } = await request(base);
@@ -1194,7 +1272,15 @@ describe('a token is good once, for one user, for a limited time', () => {
     const res = await call(base, 'apply_file_upload', { branch: DRAFT, token, destination: KB_DIR });
     expect(res.status).toBe(404);
     expect((await json<{ error: string }>(res)).error).toBe(UPLOAD_TOKEN_REFUSAL);
+    // The refusal drops the record and starts the delete without awaiting it,
+    // so a sweep run straight after no longer names the file and leaves it to
+    // that delete. Wait for the delete to land, under a deadline, instead of
+    // racing it: a readdir taken mid-`rm` failed CI on PR #457.
     await uploads.sweepNow();
+    const deadline = Date.now() + 2_000;
+    while (Date.now() < deadline && (await readdir(uploadsDir)).length > 0) {
+      await new Promise((r) => setTimeout(r, 10));
+    }
     expect(await readdir(uploadsDir)).toEqual([]);
   });
 

package/src/modules/workspace/__tests__/escape-sequences.routes.test.ts

@@ -17,6 +17,7 @@ import { WorkflowHooks } from '../../workflow/workflow-hooks.js';
 import { ToolDescriptionNotes } from '../agent-access.gate.js';
 import { SpillStore } from '../spill-store.js';
 import { DocExtractService } from '../file-readers/doc-extract.service.js';
+import { WorkerDocumentExtractor } from '../file-readers/extract-in-worker.js';
 import type { IAccessControl } from '../../access/access-control.interface.js';
 import { createMcpRoutes } from '../../mcp/mcp.routes.js';
 import { McpService } from '../../mcp/mcp.service.js';
@@ -136,7 +137,7 @@ async function startPlatform(): Promise<Platform> {
     fakeAuth,
     createToolHandlerFactory(resolve),
     new SpillStore(join(tempDir, 'spills')),
-    new DocExtractService(join(tempDir, 'doc-extract')),
+    new DocExtractService(join(tempDir, 'doc-extract'), new WorkerDocumentExtractor()),
     allowAll,
     testKbContext({ kbDirName: KB_DIR }),
     { recoveryBotEmail: 'recovery-bot@bevel.local', hooks: new WorkflowHooks(), notes: new ToolDescriptionNotes() },

package/src/modules/workspace/__tests__/file-stat-access.test.ts

@@ -16,6 +16,7 @@ import { WorkflowHooks } from '../../workflow/workflow-hooks.js';
 import { ToolDescriptionNotes } from '../../workspace/agent-access.gate.js';
 import { SpillStore } from '../spill-store.js';
 import { DocExtractService } from '../file-readers/doc-extract.service.js';
+import { WorkerDocumentExtractor } from '../file-readers/extract-in-worker.js';
 import { AccessControlService } from '../../access/access-control.service.js';
 import { NodeFs } from '../../kb-fs/node-fs.js';
 import { workspaceIdForBranch } from '../../../shared/workspace-id.js';
@@ -86,7 +87,7 @@ beforeAll(async () => {
     },
     createToolHandlerFactory(resolve),
     new SpillStore(join(tmpdir(), 'bevel-test-spills')),
-    new DocExtractService(docCache),
+    new DocExtractService(docCache, new WorkerDocumentExtractor()),
     service,
     testKbContext({ kbDirName: KB }),
     { recoveryBotEmail: 'recovery-bot@bevel.local', hooks: new WorkflowHooks(), notes: new ToolDescriptionNotes() },

package/src/modules/workspace/__tests__/git-internals.security.test.ts

@@ -28,6 +28,7 @@ import { registerWorkspaceTools } from '../workspace.tools.js';
 import { RoutineWritePolicyService } from '../routine-write-policy.js';
 import { SpillStore } from '../spill-store.js';
 import { DocExtractService } from '../file-readers/doc-extract.service.js';
+import { WorkerDocumentExtractor } from '../file-readers/extract-in-worker.js';
 import { createWorkspaceRoutes } from '../workspace.routes.js';
 import { createGitInternalsRouteGuard } from '../git-internals.middleware.js';
 import { hasGitInternalsSegment } from '../../../shared/git-internals.js';
@@ -35,11 +36,11 @@ import { createDiffRoutes } from '../../diff/diff.routes.js';
 import { DiffService } from '../../diff/diff.service.js';
 import { WorkspaceMutex } from '../../kb-fs/mutex.js';
 import { WorkspaceService } from '../workspace.service.js';
-import { WorkerPatternMatcher } from '../bounded-matcher.js';
+import { RipgrepPatternMatcher } from '../index.js';
 import { newWorkspaceService } from '../../../__tests__/workspace-service.js';
 
 /** The grep matcher this file's harness wires, closed once the file is done. */
-const patternMatcher = new WorkerPatternMatcher();
+const patternMatcher = new RipgrepPatternMatcher();
 afterAll(() => patternMatcher.close());
 
 /**
@@ -282,7 +283,7 @@ describe('workspace tools refuse the git folder', () => {
       fakeAuth,
       createToolHandlerFactory(resolve),
       new SpillStore(join(root, 'spills')),
-      new DocExtractService(join(root, 'doc-cache')),
+      new DocExtractService(join(root, 'doc-cache'), new WorkerDocumentExtractor()),
       allowAll,
       testKbContext({ kbDirName: KB }),
       { recoveryBotEmail: 'recovery-bot@bevel.local', hooks: new WorkflowHooks(), notes: new ToolDescriptionNotes() },

package/src/modules/workspace/__tests__/holds-nul.test.ts

@@ -0,0 +1,66 @@
+import { lstat, mkdtemp, rename, rm, symlink, writeFile } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { afterEach, beforeEach, describe, expect, it } from 'vitest';
+import { holdsNul } from '../nul-scan.js';
+import { PatternTooExpensiveError, type IPatternSearch } from '../pattern-matcher.contract.js';
+
+/**
+ * The unit `grep` asks whether a named text file ripgrep answered no line from
+ * is binary (`nul-scan.ts`). What a caller observes — a NUL past the first
+ * chunk makes the answer "no searchable text" — is pinned through the grep
+ * tool itself in `workspace.tools.test.ts`. Two properties stay here because
+ * no caller can stage them: the file swapped for another, or for a link,
+ * between the stat that judged it and the open that reads it is a race a tool
+ * call cannot time; and a budget already spent is told apart from ripgrep's
+ * own spending only by handing the unit a search whose budget is gone.
+ */
+let dir = '';
+const free: IPatternSearch = {
+  files: () => (async function* () {})(),
+  list: async () => [],
+  matchingLines: async (texts) => texts.map(() => []),
+  charged: (work) => work(),
+};
+
+beforeEach(async () => {
+  dir = await mkdtemp(join(tmpdir(), 'holds-nul-'));
+});
+afterEach(async () => {
+  await rm(dir, { recursive: true, force: true });
+});
+
+describe('holdsNul', () => {
+  it('finds a NUL wherever it is, past the first chunk too, and answers false for text', async () => {
+    const text = join(dir, 'text.md');
+    const late = join(dir, 'late.bin');
+    await writeFile(text, 'plain text\n'.repeat(20_000));
+    await writeFile(late, Buffer.concat([Buffer.alloc(200 * 1024, 'x'), Buffer.from([0])]));
+    expect(await holdsNul(text, await lstat(text), free)).toBe(false);
+    expect(await holdsNul(late, await lstat(late), free)).toBe(true);
+  });
+
+  it.skipIf(process.platform === 'win32')('reads nothing from a file swapped in since it was judged, nor through a link', async () => {
+    const p = join(dir, 'named.md');
+    await writeFile(p, 'text\n');
+    const judged = await lstat(p);
+    // Another file holding a NUL, moved in under the judged name.
+    await writeFile(join(dir, 'other'), Buffer.from([0]));
+    await rename(join(dir, 'other'), p);
+    expect(await holdsNul(p, judged, free)).toBe(false);
+    // A link to a file holding a NUL, under the judged name.
+    await writeFile(join(dir, 'outside'), Buffer.from([0]));
+    await rm(p);
+    await symlink(join(dir, 'outside'), p);
+    expect(await holdsNul(p, judged, free)).toBe(false);
+    expect(await holdsNul(p, await lstat(join(dir, 'outside')), free)).toBe(false);
+  });
+
+  it('runs under the search budget: a budget already spent answers for it', async () => {
+    const p = join(dir, 'named.md');
+    await writeFile(p, 'text\n');
+    const spent = new PatternTooExpensiveError('needle', 5_000);
+    const exhausted: IPatternSearch = { ...free, charged: () => Promise.reject(spent) };
+    await expect(holdsNul(p, await lstat(p), exhausted)).rejects.toBe(spent);
+  });
+});

package/src/modules/workspace/__tests__/rest-tool-arguments.test.ts

@@ -17,14 +17,15 @@ import { WorkflowHooks } from '../../workflow/workflow-hooks.js';
 import { ToolDescriptionNotes } from '../agent-access.gate.js';
 import { SpillStore } from '../spill-store.js';
 import { DocExtractService } from '../file-readers/doc-extract.service.js';
+import { WorkerDocumentExtractor } from '../file-readers/extract-in-worker.js';
 import { AccessControlService } from '../../access/access-control.service.js';
 import { NodeFs } from '../../kb-fs/node-fs.js';
 import { BRANCH_REQUIRED_MESSAGE } from '../../../shared/domain-errors.js';
 import { callLine, compileCheck, exampleArguments } from '@bevel-software/platform-mcp-core';
-import { WorkerPatternMatcher } from '../bounded-matcher.js';
+import { RipgrepPatternMatcher } from '../index.js';
 
 /** The grep matcher this file's harness wires, closed once the file is done. */
-const patternMatcher = new WorkerPatternMatcher();
+const patternMatcher = new RipgrepPatternMatcher();
 afterAll(() => patternMatcher.close());
 
 /**
@@ -91,7 +92,7 @@ beforeAll(async () => {
     },
     createToolHandlerFactory(resolve),
     new SpillStore(join(tmpdir(), 'bevel-test-spills')),
-    new DocExtractService(docCache),
+    new DocExtractService(docCache, new WorkerDocumentExtractor()),
     access,
     testKbContext({ kbDirName: KB }),
     { recoveryBotEmail: 'recovery-bot@bevel.local', hooks: new WorkflowHooks(), notes: new ToolDescriptionNotes() },
@@ -178,7 +179,7 @@ describe('POST /api/agent/tools/<name> checks the arguments against the tool', (
     expect(status).toBe(400);
     expect(body.kind).toBe('arguments-do-not-match');
     expect(body.error).toContain('"pattern" is required, and was not given.');
-    expect(body.error).toContain('pattern (string, required) — JavaScript regular expression.');
+    expect(body.error).toContain('pattern (string, required) — Regular expression, RE2-style (ripgrep): no look-around, no backreferences.');
     // Not a 200 that matched every file, which is what this surface answered
     // before the check lived in the route.
     expect(body.matches).toBeUndefined();