@link-assistant/web-capture 1.10.8 → 1.10.10

package/src/stream.js

@@ -1,6 +1,10 @@
 import fetch from 'node-fetch';
-import { pipeline } from 'stream';
 import { convertGoogleDriveUrl } from './lib.js';
+import { copyProxyResponseHeaders } from './proxy-headers.js';
+
+const STREAM_FETCH_HEADERS = {
+  'accept-encoding': 'identity',
+};
 
 export async function streamHandler(req, res) {
   const url = req.query.url;
@@ -8,36 +12,18 @@ export async function streamHandler(req, res) {
     return res.status(400).send('Missing `url` parameter');
   }
   try {
-    const response = await fetch(convertGoogleDriveUrl(url));
+    const response = await fetch(convertGoogleDriveUrl(url), {
+      headers: STREAM_FETCH_HEADERS,
+    });
     // Copy status and headers
     res.status(response.status);
-
-    // Set default content type if not present
-    const contentType = response.headers.get('content-type') || 'text/plain';
-    res.setHeader('Content-Type', contentType);
-
-    // Copy other headers
-    for (const [key, value] of response.headers.entries()) {
-      if (
-        key.toLowerCase() !== 'transfer-encoding' &&
-        key.toLowerCase() !== 'content-encoding' &&
-        key.toLowerCase() !== 'content-length'
-      ) {
-        res.setHeader(key, value);
-      }
-    }
+    copyProxyResponseHeaders(response.headers, res, {
+      preserveContentLength: true,
+    });
 
     // Stream the response body
     if (response.body) {
-      pipeline(response.body, res, (err) => {
-        if (err) {
-          console.error('Pipeline error in /stream:', err);
-          if (!res.headersSent) {
-            res.status(500);
-            res.end('Error proxying content');
-          }
-        }
-      });
+      await writeStreamResponse(response.body, res);
     } else {
       res.end();
     }
@@ -49,3 +35,24 @@ export async function streamHandler(req, res) {
     }
   }
 }
+
+async function writeStreamResponse(body, res) {
+  try {
+    for await (const chunk of body) {
+      if (!res.write(chunk)) {
+        await new Promise((resolve) => res.once('drain', resolve));
+      }
+    }
+    if (!res.writableEnded) {
+      res.end();
+    }
+  } catch (err) {
+    console.error('Upstream stream failed in /stream:', err);
+    if (!res.headersSent) {
+      res.status(500);
+      res.end('Error proxying content');
+    } else if (!res.writableEnded) {
+      res.end();
+    }
+  }
+}

package/tests/ci/detect-code-changes.test.js

@@ -0,0 +1,174 @@
+import { execFileSync } from 'child_process';
+import {
+  mkdirSync,
+  mkdtempSync,
+  readFileSync,
+  rmSync,
+  writeFileSync,
+} from 'fs';
+import { tmpdir } from 'os';
+import { dirname, join, resolve } from 'path';
+import { fileURLToPath } from 'url';
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = dirname(__filename);
+const repoRoot = resolve(__dirname, '../../..');
+const detectorPath = resolve(repoRoot, 'scripts/detect-code-changes.mjs');
+
+function run(command, args, cwd, options = {}) {
+  return execFileSync(command, args, {
+    cwd,
+    encoding: 'utf8',
+    stdio: ['ignore', 'pipe', 'pipe'],
+    ...options,
+  });
+}
+
+function git(repoPath, args) {
+  return run('git', args, repoPath);
+}
+
+function writeTrackedFile(repoPath, relativePath, contents) {
+  const filePath = join(repoPath, relativePath);
+  mkdirSync(dirname(filePath), { recursive: true });
+  writeFileSync(filePath, contents);
+}
+
+function commitAll(repoPath, message) {
+  git(repoPath, ['add', '.']);
+  git(repoPath, ['commit', '-m', message]);
+}
+
+function createMergedFeatureRepo() {
+  const repoPath = mkdtempSync(join(tmpdir(), 'web-capture-detect-'));
+
+  initializeRepo(repoPath);
+
+  writeTrackedFile(repoPath, 'README.md', '# fixture\n');
+  commitAll(repoPath, 'initial commit');
+
+  git(repoPath, ['checkout', '-b', 'feature']);
+  writeTrackedFile(repoPath, 'rust/src/lib.rs', 'pub fn capture() {}\n');
+  commitAll(repoPath, 'add rust code');
+
+  writeTrackedFile(repoPath, 'docs/notes.md', '# docs only tail commit\n');
+  commitAll(repoPath, 'document rust change');
+
+  git(repoPath, ['checkout', 'main']);
+  git(repoPath, ['merge', '--no-ff', 'feature', '-m', 'merge feature']);
+
+  return repoPath;
+}
+
+function createSingleCommitRepo() {
+  return createSingleCommitRepoWithFiles({
+    'js/src/index.js': 'export const ok = true;\n',
+  });
+}
+
+function createSingleCommitRepoWithFiles(files) {
+  const repoPath = mkdtempSync(join(tmpdir(), 'web-capture-detect-'));
+
+  initializeRepo(repoPath);
+  for (const [relativePath, contents] of Object.entries(files)) {
+    writeTrackedFile(repoPath, relativePath, contents);
+  }
+  commitAll(repoPath, 'initial commit');
+
+  return repoPath;
+}
+
+function initializeRepo(repoPath) {
+  git(repoPath, ['init']);
+  git(repoPath, ['config', 'user.email', 'ci@example.test']);
+  git(repoPath, ['config', 'user.name', 'CI Test']);
+  git(repoPath, ['checkout', '-b', 'main']);
+}
+
+function parseOutputs(outputFile) {
+  return Object.fromEntries(
+    readFileSync(outputFile, 'utf8')
+      .trim()
+      .split('\n')
+      .filter(Boolean)
+      .map((line) => {
+        const separatorIndex = line.indexOf('=');
+        return [line.slice(0, separatorIndex), line.slice(separatorIndex + 1)];
+      })
+  );
+}
+
+function runDetector(repoPath, eventName) {
+  const outputFile = join(repoPath, `github-output-${eventName}.txt`);
+  const env = {
+    ...process.env,
+    GITHUB_EVENT_NAME: eventName,
+    GITHUB_OUTPUT: outputFile,
+  };
+  delete env.GITHUB_BASE_SHA;
+  delete env.GITHUB_HEAD_SHA;
+
+  run(process.execPath, [detectorPath], repoPath, { env });
+
+  return parseOutputs(outputFile);
+}
+
+describe('detect-code-changes', () => {
+  let repoPath;
+
+  afterEach(() => {
+    if (repoPath) {
+      rmSync(repoPath, { recursive: true, force: true });
+      repoPath = undefined;
+    }
+  });
+
+  test('detects code introduced by a real merge commit pushed to main', () => {
+    repoPath = createMergedFeatureRepo();
+
+    const outputs = runDetector(repoPath, 'push');
+
+    expect(outputs['rust-code-changed']).toBe('true');
+    expect(outputs['rust-changed']).toBe('true');
+    expect(outputs['any-rust-code-changed']).toBe('true');
+    expect(outputs['any-code-changed']).toBe('true');
+    expect(outputs['docs-changed']).toBe('true');
+  });
+
+  test('keeps pull request merge commits scoped to the PR head commit', () => {
+    repoPath = createMergedFeatureRepo();
+
+    const outputs = runDetector(repoPath, 'pull_request');
+
+    expect(outputs['rust-code-changed']).toBe('false');
+    expect(outputs['rust-changed']).toBe('false');
+    expect(outputs['any-rust-code-changed']).toBe('false');
+    expect(outputs['any-code-changed']).toBe('false');
+    expect(outputs['docs-changed']).toBe('true');
+  });
+
+  test('detects files when a push only has one commit', () => {
+    repoPath = createSingleCommitRepo();
+
+    const outputs = runDetector(repoPath, 'push');
+
+    expect(outputs['js-code-changed']).toBe('true');
+    expect(outputs['js-changed']).toBe('true');
+    expect(outputs['any-js-code-changed']).toBe('true');
+    expect(outputs['any-code-changed']).toBe('true');
+  });
+
+  test('runs JS checks for CI tests without requiring a changeset', () => {
+    repoPath = createSingleCommitRepoWithFiles({
+      'js/jest.config.mjs': 'export default {};\n',
+      'js/tests/ci/workflow-policy.test.js': 'test("ok", () => {});\n',
+    });
+
+    const outputs = runDetector(repoPath, 'push');
+
+    expect(outputs['js-code-changed']).toBe('false');
+    expect(outputs['js-changed']).toBe('true');
+    expect(outputs['any-js-code-changed']).toBe('true');
+    expect(outputs['any-code-changed']).toBe('true');
+  });
+});

package/tests/ci/rust-dockerfile.test.js

@@ -0,0 +1,54 @@
+import { readFileSync } from 'fs';
+import { dirname, resolve } from 'path';
+import { fileURLToPath } from 'url';
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = dirname(__filename);
+const repoRoot = resolve(__dirname, '../../..');
+
+function parseVersion(version) {
+  return version.split('.').map((part) => Number(part));
+}
+
+function compareVersions(left, right) {
+  const leftParts = parseVersion(left);
+  const rightParts = parseVersion(right);
+  const maxLength = Math.max(leftParts.length, rightParts.length);
+
+  for (let index = 0; index < maxLength; index += 1) {
+    const leftPart = leftParts[index] || 0;
+    const rightPart = rightParts[index] || 0;
+
+    if (leftPart !== rightPart) {
+      return leftPart - rightPart;
+    }
+  }
+
+  return 0;
+}
+
+describe('Rust Dockerfile', () => {
+  test('uses a Rust builder image that satisfies Cargo.toml rust-version', () => {
+    const cargoToml = readFileSync(
+      resolve(repoRoot, 'rust/Cargo.toml'),
+      'utf8'
+    );
+    const dockerfile = readFileSync(
+      resolve(repoRoot, 'rust/Dockerfile'),
+      'utf8'
+    );
+
+    const requiredRustVersion = cargoToml.match(
+      /^rust-version = "([^"]+)"/m
+    )?.[1];
+    const dockerRustVersion = dockerfile.match(
+      /^FROM rust:(\d+(?:\.\d+){1,2})[-\s]/m
+    )?.[1];
+
+    expect(requiredRustVersion).toBeDefined();
+    expect(dockerRustVersion).toBeDefined();
+    expect(
+      compareVersions(dockerRustVersion, requiredRustVersion)
+    ).toBeGreaterThanOrEqual(0);
+  });
+});

package/tests/ci/workflow-policy.test.js

@@ -0,0 +1,63 @@
+import { readFileSync } from 'fs';
+import { dirname, resolve } from 'path';
+import { fileURLToPath } from 'url';
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = dirname(__filename);
+const repoRoot = resolve(__dirname, '../../..');
+
+const workflowFiles = [
+  '.github/workflows/js.yml',
+  '.github/workflows/rust.yml',
+  '.github/workflows/parity.yml',
+];
+
+function readRootFile(relativePath) {
+  return readFileSync(resolve(repoRoot, relativePath), 'utf8');
+}
+
+function readWorkflow(relativePath) {
+  return readRootFile(relativePath);
+}
+
+describe('workflow policy', () => {
+  test.each(workflowFiles)(
+    '%s uses release-safe concurrency cancellation',
+    (workflowFile) => {
+      const workflow = readWorkflow(workflowFile);
+
+      expect(workflow).toContain(
+        "cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}"
+      );
+      expect(workflow).not.toContain(
+        "cancel-in-progress: ${{ github.ref == 'refs/heads/main' }}"
+      );
+    }
+  );
+
+  test.each(workflowFiles)(
+    '%s uses the current checkout action',
+    (workflowFile) => {
+      const workflow = readWorkflow(workflowFile);
+
+      expect(workflow).toContain('uses: actions/checkout@v6');
+      expect(workflow).not.toContain('uses: actions/checkout@v4');
+    }
+  );
+
+  test('Rust workflow uses the current cache action', () => {
+    const workflow = readWorkflow('.github/workflows/rust.yml');
+
+    expect(workflow).toContain('uses: actions/cache@v5');
+    expect(workflow).not.toContain('uses: actions/cache@v4');
+  });
+
+  test('parity check ignores repository-level CI tests', () => {
+    const parityScript = readRootFile('scripts/check-js-rust-parity.mjs');
+
+    expect(parityScript).toContain('"js/tests/ci/"');
+    expect(parityScript).toContain(
+      'matchingFiles(files, JS_PARITY_PATHS, JS_PARITY_IGNORE_PATHS)'
+    );
+  });
+});

package/tests/e2e/docker.test.js

@@ -40,9 +40,8 @@ beforeAll(async () => {
   await new Promise((resolve) =>
     mockServer.listen(mockPort, '0.0.0.0', resolve)
   );
-  // Use host.docker.internal or the host IP to make it reachable from Docker
-  // On Linux (CI), Docker containers can reach the host via 172.17.0.1
-  mockUrl = `http://172.17.0.1:${mockPort}`;
+  // docker-compose.yml maps this hostname to the host gateway on Linux CI.
+  mockUrl = `http://host.docker.internal:${mockPort}`;
 
   console.log('Checking if Docker service is already running...');
   const alreadyRunning = await isServiceRunning();

package/tests/e2e/process.test.js

@@ -1,13 +1,13 @@
 import { spawn } from 'child_process';
-import http from 'http';
 import fetch from 'node-fetch';
 import getPort from 'get-port';
 import path from 'path';
 
 const WAIT_FOR_READY = 5000; // ms
 let serverProcess;
+let serverOutput = '';
 let baseUrl;
-let mockServer;
+let mockProcess;
 let mockUrl;
 
 const MOCK_HTML = `<!DOCTYPE html>
@@ -17,17 +17,13 @@ const MOCK_HTML = `<!DOCTYPE html>
 </html>`;
 
 beforeAll(async () => {
-  // Start a local mock server to avoid depending on external network
   const mockPort = await getPort();
-  mockServer = http.createServer((req, res) => {
-    res.writeHead(200, { 'Content-Type': 'text/html; charset=utf-8' });
-    res.end(MOCK_HTML);
-  });
-  await new Promise((resolve) => mockServer.listen(mockPort, resolve));
-  mockUrl = `http://localhost:${mockPort}`;
+  mockProcess = startMockServer(mockPort);
+  await waitForProcessOutput(mockProcess, 'mock fixture server listening');
+  mockUrl = `http://127.0.0.1:${mockPort}`;
 
   const port = await getPort();
-  baseUrl = `http://localhost:${port}`;
+  baseUrl = `http://127.0.0.1:${port}`;
 
   serverProcess = spawn(
     'node',
@@ -37,6 +33,9 @@ beforeAll(async () => {
       stdio: ['ignore', 'pipe', 'pipe'],
     }
   );
+  collectProcessOutput(serverProcess, (chunk) => {
+    serverOutput += chunk;
+  });
 
   // Wait for the server to be ready (simple delay or poll)
   await new Promise((resolve, reject) => {
@@ -79,8 +78,8 @@ afterAll(() => {
   if (serverProcess) {
     serverProcess.kill();
   }
-  if (mockServer) {
-    mockServer.close();
+  if (mockProcess) {
+    mockProcess.kill();
   }
 });
 
@@ -89,8 +88,7 @@ describe('E2E: Web Capture Microservice', () => {
     const res = await fetch(
       `${baseUrl}/html?url=${encodeURIComponent(mockUrl)}`
     );
-    expect(res.status).toBe(200);
-    const text = await res.text();
+    const text = await expectTextResponse(res);
     expect(text).toMatch(/<html/i);
   });
 
@@ -98,8 +96,7 @@ describe('E2E: Web Capture Microservice', () => {
     const res = await fetch(
       `${baseUrl}/markdown?url=${encodeURIComponent(mockUrl)}`
     );
-    expect(res.status).toBe(200);
-    const text = await res.text();
+    const text = await expectTextResponse(res);
     expect(text).toMatch(/example/i);
   });
 
@@ -107,7 +104,7 @@ describe('E2E: Web Capture Microservice', () => {
     const res = await fetch(
       `${baseUrl}/image?url=${encodeURIComponent(mockUrl)}`
     );
-    expect(res.status).toBe(200);
+    await expectStatusOk(res);
     expect(res.headers.get('content-type')).toMatch(/^image\/png/);
     const buf = Buffer.from(await res.arrayBuffer());
     // PNG signature check
@@ -122,9 +119,7 @@ describe('E2E: Web Capture Microservice', () => {
     const res = await fetch(
       `${baseUrl}/stream?url=${encodeURIComponent(mockUrl)}`
     );
-    expect(res.status).toBe(200);
-    // Get the response as text
-    const text = await res.text();
+    const text = await expectTextResponse(res);
     expect(text).toMatch(/<html/i);
     expect(text).toMatch(/Example Domain/i);
   }, 20000);
@@ -133,9 +128,87 @@ describe('E2E: Web Capture Microservice', () => {
     const res = await fetch(
       `${baseUrl}/fetch?url=${encodeURIComponent(mockUrl)}`
     );
-    expect(res.status).toBe(200);
-    const text = await res.text();
+    const text = await expectTextResponse(res);
     expect(text).toMatch(/<html/i);
     expect(text).toMatch(/Example Domain/i);
   }, 20000);
 });
+
+function startMockServer(port) {
+  const script = `
+    const http = require('http');
+    const html = ${JSON.stringify(MOCK_HTML)};
+    const server = http.createServer((req, res) => {
+      res.writeHead(200, {
+        'Content-Type': 'text/html; charset=utf-8',
+        'Content-Length': Buffer.byteLength(html),
+      });
+      res.end(html);
+    });
+    server.listen(${port}, '127.0.0.1', () => {
+      console.log('mock fixture server listening');
+    });
+    process.on('SIGTERM', () => server.close(() => process.exit(0)));
+    process.on('SIGINT', () => server.close(() => process.exit(0)));
+  `;
+  return spawn('node', ['-e', script], {
+    stdio: ['ignore', 'pipe', 'pipe'],
+  });
+}
+
+function collectProcessOutput(child, onChunk) {
+  child.stdout.on('data', (data) => onChunk(data.toString()));
+  child.stderr.on('data', (data) => onChunk(data.toString()));
+}
+
+async function waitForProcessOutput(child, expected) {
+  let output = '';
+  collectProcessOutput(child, (chunk) => {
+    output += chunk;
+  });
+  await new Promise((resolve, reject) => {
+    const timeout = setTimeout(() => {
+      reject(
+        new Error(`Timed out waiting for "${expected}". Output:\n${output}`)
+      );
+    }, WAIT_FOR_READY);
+    child.stdout.on('data', (data) => {
+      if (data.toString().includes(expected)) {
+        clearTimeout(timeout);
+        resolve();
+      }
+    });
+    child.on('exit', (code, signal) => {
+      clearTimeout(timeout);
+      reject(new Error(`Process exited with code ${code}, signal ${signal}`));
+    });
+  });
+}
+
+async function expectTextResponse(res) {
+  let text;
+  try {
+    text = await res.text();
+  } catch (error) {
+    throw new Error(
+      `${error.message}\nServer output:\n${serverOutput || '(empty)'}`
+    );
+  }
+  expectResponseStatus(res.status, text);
+  return text;
+}
+
+async function expectStatusOk(res) {
+  if (res.status === 200) {
+    return;
+  }
+  expectResponseStatus(res.status, await res.text());
+}
+
+function expectResponseStatus(status, body) {
+  if (status !== 200) {
+    throw new Error(
+      `Expected HTTP 200, received ${status}\nBody:\n${body}\nServer output:\n${serverOutput}`
+    );
+  }
+}

package/tests/unit/cli.test.js

@@ -43,7 +43,10 @@ function startFixtureServer({
 } = {}) {
   return new Promise((resolve, reject) => {
     const server = http.createServer((req, res) => {
-      res.writeHead(200, { 'Content-Type': contentType });
+      res.writeHead(200, {
+        'Content-Type': contentType,
+        'Content-Length': Buffer.byteLength(body),
+      });
       res.end(body);
     });
 
@@ -351,5 +354,46 @@ describe('CLI', () => {
       });
       expect(body.diagnostics.sourceUrl).toContain('en.wikipedia.org');
     }, 20000);
+
+    test('emits shared-dialog demo_memory from the shared-dialog subcommand', async () => {
+      const fixturePath = resolve(
+        __dirname,
+        '../../../docs/case-studies/issue-141/raw-data/chatgpt-share-6a3825b9.html'
+      );
+      const body = readFileSync(fixturePath, 'utf-8');
+      const { server, url } = await startFixtureServer({ body });
+      const shareUrl = url.replace(
+        '/article',
+        '/share/6a3825b9-8de4-83ee-9c24-52fd1eb38d24'
+      );
+
+      try {
+        const result = await runCli(
+          [
+            'shared-dialog',
+            shareUrl,
+            '--format',
+            'demo-memory',
+            '--output',
+            '-',
+          ],
+          {
+            cwd: resolve(__dirname, '../..'),
+          }
+        );
+
+        expect(result.code).toBe(0);
+        expect(result.stdout.startsWith('demo_memory')).toBe(true);
+        expect(result.stdout).toContain('demo_memory');
+        expect(result.stdout.match(/\n {2}event "/g)).toHaveLength(4);
+        expect(result.stdout).toContain('role "user"');
+        expect(result.stdout).toContain('role "assistant"');
+        expect(result.stdout).toContain(
+          'conversationTitle "Infinite loop script"'
+        );
+      } finally {
+        await stopFixtureServer(server);
+      }
+    }, 20000);
   });
 });

package/tests/unit/fetch-html.test.js

@@ -0,0 +1,24 @@
+import nock from 'nock';
+import { fetchHtml } from '../../src/lib.js';
+
+afterEach(() => {
+  nock.cleanAll();
+});
+
+describe('fetchHtml', () => {
+  it('requests identity encoding for document HTML fetches', async () => {
+    nock('https://example.com', {
+      reqheaders: {
+        'accept-encoding': 'identity',
+      },
+    })
+      .get('/article')
+      .reply(200, '<html><body>ok</body></html>', {
+        'content-type': 'text/html; charset=utf-8',
+      });
+
+    await expect(fetchHtml('https://example.com/article')).resolves.toContain(
+      'ok'
+    );
+  });
+});

package/tests/unit/gdocs-export-headers.test.js

@@ -0,0 +1,28 @@
+import nock from 'nock';
+import { fetchGoogleDocAsMarkdown } from '../../src/gdocs.js';
+
+afterEach(() => {
+  nock.cleanAll();
+});
+
+describe('Google Docs public export headers', () => {
+  it('requests identity encoding for public export fetches', async () => {
+    nock('https://docs.google.com', {
+      reqheaders: {
+        'accept-encoding': 'identity',
+      },
+    })
+      .get('/document/d/export-doc/export')
+      .query({ format: 'html' })
+      .reply(200, '<html><body><h1>Export Doc</h1></body></html>', {
+        'content-type': 'text/html; charset=utf-8',
+      });
+
+    const result = await fetchGoogleDocAsMarkdown(
+      'https://docs.google.com/document/d/export-doc/edit'
+    );
+
+    expect(result.markdown).toContain('Export Doc');
+    expect(nock.isDone()).toBe(true);
+  });
+});