@link-assistant/web-capture 1.10.9 → 1.10.11

package/src/stream.js

@@ -1,6 +1,10 @@
 import fetch from 'node-fetch';
-import { pipeline } from 'stream';
 import { convertGoogleDriveUrl } from './lib.js';
+import { copyProxyResponseHeaders } from './proxy-headers.js';
+
+const STREAM_FETCH_HEADERS = {
+  'accept-encoding': 'identity',
+};
 
 export async function streamHandler(req, res) {
   const url = req.query.url;
@@ -8,36 +12,18 @@ export async function streamHandler(req, res) {
     return res.status(400).send('Missing `url` parameter');
   }
   try {
-    const response = await fetch(convertGoogleDriveUrl(url));
+    const response = await fetch(convertGoogleDriveUrl(url), {
+      headers: STREAM_FETCH_HEADERS,
+    });
     // Copy status and headers
     res.status(response.status);
-
-    // Set default content type if not present
-    const contentType = response.headers.get('content-type') || 'text/plain';
-    res.setHeader('Content-Type', contentType);
-
-    // Copy other headers
-    for (const [key, value] of response.headers.entries()) {
-      if (
-        key.toLowerCase() !== 'transfer-encoding' &&
-        key.toLowerCase() !== 'content-encoding' &&
-        key.toLowerCase() !== 'content-length'
-      ) {
-        res.setHeader(key, value);
-      }
-    }
+    copyProxyResponseHeaders(response.headers, res, {
+      preserveContentLength: true,
+    });
 
     // Stream the response body
     if (response.body) {
-      pipeline(response.body, res, (err) => {
-        if (err) {
-          console.error('Pipeline error in /stream:', err);
-          if (!res.headersSent) {
-            res.status(500);
-            res.end('Error proxying content');
-          }
-        }
-      });
+      await writeStreamResponse(response.body, res);
     } else {
       res.end();
     }
@@ -49,3 +35,24 @@ export async function streamHandler(req, res) {
     }
   }
 }
+
+async function writeStreamResponse(body, res) {
+  try {
+    for await (const chunk of body) {
+      if (!res.write(chunk)) {
+        await new Promise((resolve) => res.once('drain', resolve));
+      }
+    }
+    if (!res.writableEnded) {
+      res.end();
+    }
+  } catch (err) {
+    console.error('Upstream stream failed in /stream:', err);
+    if (!res.headersSent) {
+      res.status(500);
+      res.end('Error proxying content');
+    } else if (!res.writableEnded) {
+      res.end();
+    }
+  }
+}

package/tests/e2e/docker.test.js

@@ -40,9 +40,8 @@ beforeAll(async () => {
   await new Promise((resolve) =>
     mockServer.listen(mockPort, '0.0.0.0', resolve)
   );
-  // Use host.docker.internal or the host IP to make it reachable from Docker
-  // On Linux (CI), Docker containers can reach the host via 172.17.0.1
-  mockUrl = `http://172.17.0.1:${mockPort}`;
+  // docker-compose.yml maps this hostname to the host gateway on Linux CI.
+  mockUrl = `http://host.docker.internal:${mockPort}`;
 
   console.log('Checking if Docker service is already running...');
   const alreadyRunning = await isServiceRunning();

package/tests/e2e/process.test.js

@@ -1,13 +1,13 @@
 import { spawn } from 'child_process';
-import http from 'http';
 import fetch from 'node-fetch';
 import getPort from 'get-port';
 import path from 'path';
 
 const WAIT_FOR_READY = 5000; // ms
 let serverProcess;
+let serverOutput = '';
 let baseUrl;
-let mockServer;
+let mockProcess;
 let mockUrl;
 
 const MOCK_HTML = `<!DOCTYPE html>
@@ -17,17 +17,13 @@ const MOCK_HTML = `<!DOCTYPE html>
 </html>`;
 
 beforeAll(async () => {
-  // Start a local mock server to avoid depending on external network
   const mockPort = await getPort();
-  mockServer = http.createServer((req, res) => {
-    res.writeHead(200, { 'Content-Type': 'text/html; charset=utf-8' });
-    res.end(MOCK_HTML);
-  });
-  await new Promise((resolve) => mockServer.listen(mockPort, resolve));
-  mockUrl = `http://localhost:${mockPort}`;
+  mockProcess = startMockServer(mockPort);
+  await waitForProcessOutput(mockProcess, 'mock fixture server listening');
+  mockUrl = `http://127.0.0.1:${mockPort}`;
 
   const port = await getPort();
-  baseUrl = `http://localhost:${port}`;
+  baseUrl = `http://127.0.0.1:${port}`;
 
   serverProcess = spawn(
     'node',
@@ -37,6 +33,9 @@ beforeAll(async () => {
       stdio: ['ignore', 'pipe', 'pipe'],
     }
   );
+  collectProcessOutput(serverProcess, (chunk) => {
+    serverOutput += chunk;
+  });
 
   // Wait for the server to be ready (simple delay or poll)
   await new Promise((resolve, reject) => {
@@ -79,8 +78,8 @@ afterAll(() => {
   if (serverProcess) {
     serverProcess.kill();
   }
-  if (mockServer) {
-    mockServer.close();
+  if (mockProcess) {
+    mockProcess.kill();
   }
 });
 
@@ -89,8 +88,7 @@ describe('E2E: Web Capture Microservice', () => {
     const res = await fetch(
       `${baseUrl}/html?url=${encodeURIComponent(mockUrl)}`
     );
-    expect(res.status).toBe(200);
-    const text = await res.text();
+    const text = await expectTextResponse(res);
     expect(text).toMatch(/<html/i);
   });
 
@@ -98,8 +96,7 @@ describe('E2E: Web Capture Microservice', () => {
     const res = await fetch(
       `${baseUrl}/markdown?url=${encodeURIComponent(mockUrl)}`
     );
-    expect(res.status).toBe(200);
-    const text = await res.text();
+    const text = await expectTextResponse(res);
     expect(text).toMatch(/example/i);
   });
 
@@ -107,7 +104,7 @@ describe('E2E: Web Capture Microservice', () => {
     const res = await fetch(
       `${baseUrl}/image?url=${encodeURIComponent(mockUrl)}`
     );
-    expect(res.status).toBe(200);
+    await expectStatusOk(res);
     expect(res.headers.get('content-type')).toMatch(/^image\/png/);
     const buf = Buffer.from(await res.arrayBuffer());
     // PNG signature check
@@ -122,9 +119,7 @@ describe('E2E: Web Capture Microservice', () => {
     const res = await fetch(
       `${baseUrl}/stream?url=${encodeURIComponent(mockUrl)}`
     );
-    expect(res.status).toBe(200);
-    // Get the response as text
-    const text = await res.text();
+    const text = await expectTextResponse(res);
     expect(text).toMatch(/<html/i);
     expect(text).toMatch(/Example Domain/i);
   }, 20000);
@@ -133,9 +128,87 @@ describe('E2E: Web Capture Microservice', () => {
     const res = await fetch(
       `${baseUrl}/fetch?url=${encodeURIComponent(mockUrl)}`
     );
-    expect(res.status).toBe(200);
-    const text = await res.text();
+    const text = await expectTextResponse(res);
     expect(text).toMatch(/<html/i);
     expect(text).toMatch(/Example Domain/i);
   }, 20000);
 });
+
+function startMockServer(port) {
+  const script = `
+    const http = require('http');
+    const html = ${JSON.stringify(MOCK_HTML)};
+    const server = http.createServer((req, res) => {
+      res.writeHead(200, {
+        'Content-Type': 'text/html; charset=utf-8',
+        'Content-Length': Buffer.byteLength(html),
+      });
+      res.end(html);
+    });
+    server.listen(${port}, '127.0.0.1', () => {
+      console.log('mock fixture server listening');
+    });
+    process.on('SIGTERM', () => server.close(() => process.exit(0)));
+    process.on('SIGINT', () => server.close(() => process.exit(0)));
+  `;
+  return spawn('node', ['-e', script], {
+    stdio: ['ignore', 'pipe', 'pipe'],
+  });
+}
+
+function collectProcessOutput(child, onChunk) {
+  child.stdout.on('data', (data) => onChunk(data.toString()));
+  child.stderr.on('data', (data) => onChunk(data.toString()));
+}
+
+async function waitForProcessOutput(child, expected) {
+  let output = '';
+  collectProcessOutput(child, (chunk) => {
+    output += chunk;
+  });
+  await new Promise((resolve, reject) => {
+    const timeout = setTimeout(() => {
+      reject(
+        new Error(`Timed out waiting for "${expected}". Output:\n${output}`)
+      );
+    }, WAIT_FOR_READY);
+    child.stdout.on('data', (data) => {
+      if (data.toString().includes(expected)) {
+        clearTimeout(timeout);
+        resolve();
+      }
+    });
+    child.on('exit', (code, signal) => {
+      clearTimeout(timeout);
+      reject(new Error(`Process exited with code ${code}, signal ${signal}`));
+    });
+  });
+}
+
+async function expectTextResponse(res) {
+  let text;
+  try {
+    text = await res.text();
+  } catch (error) {
+    throw new Error(
+      `${error.message}\nServer output:\n${serverOutput || '(empty)'}`
+    );
+  }
+  expectResponseStatus(res.status, text);
+  return text;
+}
+
+async function expectStatusOk(res) {
+  if (res.status === 200) {
+    return;
+  }
+  expectResponseStatus(res.status, await res.text());
+}
+
+function expectResponseStatus(status, body) {
+  if (status !== 200) {
+    throw new Error(
+      `Expected HTTP 200, received ${status}\nBody:\n${body}\nServer output:\n${serverOutput}`
+    );
+  }
+}

package/tests/unit/cli.test.js

@@ -43,7 +43,10 @@ function startFixtureServer({
 } = {}) {
   return new Promise((resolve, reject) => {
     const server = http.createServer((req, res) => {
-      res.writeHead(200, { 'Content-Type': contentType });
+      res.writeHead(200, {
+        'Content-Type': contentType,
+        'Content-Length': Buffer.byteLength(body),
+      });
       res.end(body);
     });
 
@@ -351,5 +354,46 @@ describe('CLI', () => {
       });
       expect(body.diagnostics.sourceUrl).toContain('en.wikipedia.org');
     }, 20000);
+
+    test('emits shared-dialog demo_memory from the shared-dialog subcommand', async () => {
+      const fixturePath = resolve(
+        __dirname,
+        '../../../docs/case-studies/issue-141/raw-data/chatgpt-share-6a3825b9.html'
+      );
+      const body = readFileSync(fixturePath, 'utf-8');
+      const { server, url } = await startFixtureServer({ body });
+      const shareUrl = url.replace(
+        '/article',
+        '/share/6a3825b9-8de4-83ee-9c24-52fd1eb38d24'
+      );
+
+      try {
+        const result = await runCli(
+          [
+            'shared-dialog',
+            shareUrl,
+            '--format',
+            'demo-memory',
+            '--output',
+            '-',
+          ],
+          {
+            cwd: resolve(__dirname, '../..'),
+          }
+        );
+
+        expect(result.code).toBe(0);
+        expect(result.stdout.startsWith('demo_memory')).toBe(true);
+        expect(result.stdout).toContain('demo_memory');
+        expect(result.stdout.match(/\n {2}event "/g)).toHaveLength(4);
+        expect(result.stdout).toContain('role "user"');
+        expect(result.stdout).toContain('role "assistant"');
+        expect(result.stdout).toContain(
+          'conversationTitle "Infinite loop script"'
+        );
+      } finally {
+        await stopFixtureServer(server);
+      }
+    }, 20000);
   });
 });

package/tests/unit/fetch-html.test.js

@@ -0,0 +1,24 @@
+import nock from 'nock';
+import { fetchHtml } from '../../src/lib.js';
+
+afterEach(() => {
+  nock.cleanAll();
+});
+
+describe('fetchHtml', () => {
+  it('requests identity encoding for document HTML fetches', async () => {
+    nock('https://example.com', {
+      reqheaders: {
+        'accept-encoding': 'identity',
+      },
+    })
+      .get('/article')
+      .reply(200, '<html><body>ok</body></html>', {
+        'content-type': 'text/html; charset=utf-8',
+      });
+
+    await expect(fetchHtml('https://example.com/article')).resolves.toContain(
+      'ok'
+    );
+  });
+});

package/tests/unit/gdocs-export-headers.test.js

@@ -0,0 +1,28 @@
+import nock from 'nock';
+import { fetchGoogleDocAsMarkdown } from '../../src/gdocs.js';
+
+afterEach(() => {
+  nock.cleanAll();
+});
+
+describe('Google Docs public export headers', () => {
+  it('requests identity encoding for public export fetches', async () => {
+    nock('https://docs.google.com', {
+      reqheaders: {
+        'accept-encoding': 'identity',
+      },
+    })
+      .get('/document/d/export-doc/export')
+      .query({ format: 'html' })
+      .reply(200, '<html><body><h1>Export Doc</h1></body></html>', {
+        'content-type': 'text/html; charset=utf-8',
+      });
+
+    const result = await fetchGoogleDocAsMarkdown(
+      'https://docs.google.com/document/d/export-doc/edit'
+    );
+
+    expect(result.markdown).toContain('Export Doc');
+    expect(nock.isDone()).toBe(true);
+  });
+});

package/tests/unit/github.test.js

@@ -1,4 +1,6 @@
+import nock from 'nock';
 import {
+  fetchGithubRepositorySnapshot,
   formatGithubRepositoryMarkdown,
   formatGithubRepositoryText,
   getGithubRepositoryTextFilename,
@@ -42,6 +44,10 @@ const SNAPSHOT = {
   },
 };
 
+afterEach(() => {
+  nock.cleanAll();
+});
+
 describe('GitHub repository URLs', () => {
   it('detects plain GitHub repository pages', () => {
     expect(
@@ -70,6 +76,59 @@ describe('GitHub repository URLs', () => {
   });
 });
 
+describe('GitHub repository snapshot fetching', () => {
+  it('requests identity encoding from GitHub JSON and raw README fetches', async () => {
+    const owner = 'octocat';
+    const repo = 'Hello-World';
+    const readme = '# Hello World\n\nRaw README content.';
+
+    const githubApi = nock('https://api.github.com', {
+      reqheaders: {
+        'accept-encoding': 'identity',
+      },
+    })
+      .get(`/repos/${owner}/${repo}`)
+      .reply(200, {
+        full_name: `${owner}/${repo}`,
+        html_url: `https://github.com/${owner}/${repo}`,
+        default_branch: 'master',
+      })
+      .get(`/repos/${owner}/${repo}/readme`)
+      .query({ ref: 'master' })
+      .reply(200, {
+        name: 'README.md',
+        path: 'README.md',
+        download_url: `https://raw.githubusercontent.com/${owner}/${repo}/master/README.md`,
+      })
+      .get(`/repos/${owner}/${repo}/contents`)
+      .query({ ref: 'master' })
+      .reply(200, [
+        {
+          name: 'README.md',
+          path: 'README.md',
+          type: 'file',
+          size: readme.length,
+        },
+      ]);
+
+    const rawGithub = nock('https://raw.githubusercontent.com', {
+      reqheaders: {
+        'accept-encoding': 'identity',
+      },
+    })
+      .get(`/${owner}/${repo}/master/README.md`)
+      .reply(200, readme);
+
+    const snapshot = await fetchGithubRepositorySnapshot(
+      `https://github.com/${owner}/${repo}`
+    );
+
+    expect(snapshot.readme.content).toBe(readme);
+    expect(githubApi.isDone()).toBe(true);
+    expect(rawGithub.isDone()).toBe(true);
+  });
+});
+
 describe('GitHub repository snapshot formatting', () => {
   it('formats a compact markdown snapshot with metadata, file tree, and README', () => {
     const markdown = formatGithubRepositoryMarkdown(SNAPSHOT);

package/tests/unit/package-runtime-dependencies.test.js

@@ -10,6 +10,25 @@ function readJson(relativePath) {
   return JSON.parse(readFileSync(resolve(jsRoot, relativePath), 'utf8'));
 }
 
+function collectProductionDependencies(packageLock, packageName) {
+  const visited = new Set();
+
+  function visit(name) {
+    if (visited.has(name)) {
+      return;
+    }
+    visited.add(name);
+
+    const entry = packageLock.packages[`node_modules/${name}`];
+    for (const dependency of Object.keys(entry?.dependencies || {})) {
+      visit(dependency);
+    }
+  }
+
+  visit(packageName);
+  return visited;
+}
+
 const runtimeImports = [
   {
     packageName: 'turndown-plugin-gfm',
@@ -18,6 +37,13 @@ const runtimeImports = [
 ];
 
 describe('runtime package dependencies', () => {
+  test('archiver production dependencies do not include deprecated glob', () => {
+    const packageLock = readJson('package-lock.json');
+    const dependencies = collectProductionDependencies(packageLock, 'archiver');
+
+    expect([...dependencies]).not.toContain('glob');
+  });
+
   test.each(runtimeImports)(
     '$packageName is declared as a production dependency',
     ({ packageName, sourceFile }) => {

package/tests/unit/proxy-headers.test.js

@@ -0,0 +1,100 @@
+import {
+  copyProxyResponseHeaders,
+  getProxyHeaderEntries,
+} from '../../src/proxy-headers.js';
+
+describe('proxy header helpers', () => {
+  it('removes hop-by-hop headers and headers named by Connection', () => {
+    const entries = getProxyHeaderEntries(
+      headers({
+        'Content-Type': 'text/html',
+        Connection: 'keep-alive, X-Internal-Hop',
+        'Keep-Alive': 'timeout=5',
+        TE: 'trailers',
+        Trailer: 'Expires',
+        Upgrade: 'websocket',
+        'Proxy-Authenticate': 'Basic',
+        'Proxy-Authorization': 'Basic token',
+        'Transfer-Encoding': 'chunked',
+        'X-Internal-Hop': 'remove me',
+        'X-End-To-End': 'keep me',
+      })
+    );
+
+    expect(Object.fromEntries(entries)).toEqual({
+      'content-type': 'text/html',
+      'x-end-to-end': 'keep me',
+    });
+  });
+
+  it('preserves content length only for unencoded streaming responses', () => {
+    expect(
+      Object.fromEntries(
+        getProxyHeaderEntries(
+          headers({
+            'Content-Type': 'text/plain',
+            'Content-Length': '12',
+          }),
+          { preserveContentLength: true }
+        )
+      )
+    ).toEqual({
+      'content-type': 'text/plain',
+      'content-length': '12',
+    });
+
+    expect(
+      Object.fromEntries(
+        getProxyHeaderEntries(
+          headers({
+            'Content-Type': 'text/plain',
+            'Content-Length': '12',
+            'Content-Encoding': 'gzip',
+          }),
+          { preserveContentLength: true }
+        )
+      )
+    ).toEqual({
+      'content-type': 'text/plain',
+    });
+  });
+
+  it('sets a default content type while copying safe headers', () => {
+    const res = responseRecorder();
+
+    copyProxyResponseHeaders(
+      headers({
+        ETag: '"abc"',
+      }),
+      res
+    );
+
+    expect(res.headers).toEqual({
+      'content-type': 'text/plain',
+      etag: '"abc"',
+    });
+  });
+});
+
+function headers(values) {
+  const map = new Map(
+    Object.entries(values).map(([key, value]) => [key.toLowerCase(), value])
+  );
+  return {
+    get(key) {
+      return map.get(key.toLowerCase()) || null;
+    },
+    entries() {
+      return map.entries();
+    },
+  };
+}
+
+function responseRecorder() {
+  return {
+    headers: {},
+    setHeader(key, value) {
+      this.headers[key.toLowerCase()] = value;
+    },
+  };
+}