@link-assistant/web-capture 1.10.8 → 1.10.10

package/tests/unit/github.test.js

@@ -1,4 +1,6 @@
+import nock from 'nock';
 import {
+  fetchGithubRepositorySnapshot,
   formatGithubRepositoryMarkdown,
   formatGithubRepositoryText,
   getGithubRepositoryTextFilename,
@@ -42,6 +44,10 @@ const SNAPSHOT = {
   },
 };
 
+afterEach(() => {
+  nock.cleanAll();
+});
+
 describe('GitHub repository URLs', () => {
   it('detects plain GitHub repository pages', () => {
     expect(
@@ -70,6 +76,59 @@ describe('GitHub repository URLs', () => {
   });
 });
 
+describe('GitHub repository snapshot fetching', () => {
+  it('requests identity encoding from GitHub JSON and raw README fetches', async () => {
+    const owner = 'octocat';
+    const repo = 'Hello-World';
+    const readme = '# Hello World\n\nRaw README content.';
+
+    const githubApi = nock('https://api.github.com', {
+      reqheaders: {
+        'accept-encoding': 'identity',
+      },
+    })
+      .get(`/repos/${owner}/${repo}`)
+      .reply(200, {
+        full_name: `${owner}/${repo}`,
+        html_url: `https://github.com/${owner}/${repo}`,
+        default_branch: 'master',
+      })
+      .get(`/repos/${owner}/${repo}/readme`)
+      .query({ ref: 'master' })
+      .reply(200, {
+        name: 'README.md',
+        path: 'README.md',
+        download_url: `https://raw.githubusercontent.com/${owner}/${repo}/master/README.md`,
+      })
+      .get(`/repos/${owner}/${repo}/contents`)
+      .query({ ref: 'master' })
+      .reply(200, [
+        {
+          name: 'README.md',
+          path: 'README.md',
+          type: 'file',
+          size: readme.length,
+        },
+      ]);
+
+    const rawGithub = nock('https://raw.githubusercontent.com', {
+      reqheaders: {
+        'accept-encoding': 'identity',
+      },
+    })
+      .get(`/${owner}/${repo}/master/README.md`)
+      .reply(200, readme);
+
+    const snapshot = await fetchGithubRepositorySnapshot(
+      `https://github.com/${owner}/${repo}`
+    );
+
+    expect(snapshot.readme.content).toBe(readme);
+    expect(githubApi.isDone()).toBe(true);
+    expect(rawGithub.isDone()).toBe(true);
+  });
+});
+
 describe('GitHub repository snapshot formatting', () => {
   it('formats a compact markdown snapshot with metadata, file tree, and README', () => {
     const markdown = formatGithubRepositoryMarkdown(SNAPSHOT);

package/tests/unit/proxy-headers.test.js

@@ -0,0 +1,100 @@
+import {
+  copyProxyResponseHeaders,
+  getProxyHeaderEntries,
+} from '../../src/proxy-headers.js';
+
+describe('proxy header helpers', () => {
+  it('removes hop-by-hop headers and headers named by Connection', () => {
+    const entries = getProxyHeaderEntries(
+      headers({
+        'Content-Type': 'text/html',
+        Connection: 'keep-alive, X-Internal-Hop',
+        'Keep-Alive': 'timeout=5',
+        TE: 'trailers',
+        Trailer: 'Expires',
+        Upgrade: 'websocket',
+        'Proxy-Authenticate': 'Basic',
+        'Proxy-Authorization': 'Basic token',
+        'Transfer-Encoding': 'chunked',
+        'X-Internal-Hop': 'remove me',
+        'X-End-To-End': 'keep me',
+      })
+    );
+
+    expect(Object.fromEntries(entries)).toEqual({
+      'content-type': 'text/html',
+      'x-end-to-end': 'keep me',
+    });
+  });
+
+  it('preserves content length only for unencoded streaming responses', () => {
+    expect(
+      Object.fromEntries(
+        getProxyHeaderEntries(
+          headers({
+            'Content-Type': 'text/plain',
+            'Content-Length': '12',
+          }),
+          { preserveContentLength: true }
+        )
+      )
+    ).toEqual({
+      'content-type': 'text/plain',
+      'content-length': '12',
+    });
+
+    expect(
+      Object.fromEntries(
+        getProxyHeaderEntries(
+          headers({
+            'Content-Type': 'text/plain',
+            'Content-Length': '12',
+            'Content-Encoding': 'gzip',
+          }),
+          { preserveContentLength: true }
+        )
+      )
+    ).toEqual({
+      'content-type': 'text/plain',
+    });
+  });
+
+  it('sets a default content type while copying safe headers', () => {
+    const res = responseRecorder();
+
+    copyProxyResponseHeaders(
+      headers({
+        ETag: '"abc"',
+      }),
+      res
+    );
+
+    expect(res.headers).toEqual({
+      'content-type': 'text/plain',
+      etag: '"abc"',
+    });
+  });
+});
+
+function headers(values) {
+  const map = new Map(
+    Object.entries(values).map(([key, value]) => [key.toLowerCase(), value])
+  );
+  return {
+    get(key) {
+      return map.get(key.toLowerCase()) || null;
+    },
+    entries() {
+      return map.entries();
+    },
+  };
+}
+
+function responseRecorder() {
+  return {
+    headers: {},
+    setHeader(key, value) {
+      this.headers[key.toLowerCase()] = value;
+    },
+  };
+}

package/tests/unit/shared-dialog.test.js

@@ -0,0 +1,173 @@
+import { readFileSync } from 'node:fs';
+import { dirname, resolve } from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+import { jest } from '@jest/globals';
+
+import {
+  captureSharedDialog,
+  formatSharedDialogAsDemoMemory,
+  formatSharedDialogAsMarkdown,
+  formatSharedDialogAsMetaLanguage,
+  parseSharedDialog,
+} from '../../src/shared-dialog.js';
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = dirname(__filename);
+const repoRoot = resolve(__dirname, '../../..');
+
+const CHATGPT_SHARE_URL =
+  'https://chatgpt.com/share/6a3825b9-8de4-83ee-9c24-52fd1eb38d24';
+const GOOGLE_AI_MODE_URL = 'https://share.google/aimode/VG0HhpnAXrBkC0QgP';
+
+function readCaseStudyFixture(name) {
+  return readFileSync(
+    resolve(repoRoot, 'docs/case-studies/issue-141/raw-data', name),
+    'utf-8'
+  );
+}
+
+describe('shared-dialog module (#141)', () => {
+  it('extracts the four visible ChatGPT shared-dialog turns', () => {
+    const capture = parseSharedDialog(
+      readCaseStudyFixture('chatgpt-share-6a3825b9.html'),
+      {
+        sourceUrl: CHATGPT_SHARE_URL,
+        captureMethod: 'static_http',
+      }
+    );
+
+    expect(capture).toMatchObject({
+      provider: 'chatgpt',
+      sourceUrl: CHATGPT_SHARE_URL,
+      captureMethod: 'static_http',
+      status: 'ok',
+      conversationId: '6a3825b9-8de4-83ee-9c24-52fd1eb38d24',
+      title: 'Infinite loop script',
+    });
+    expect(capture.turns).toHaveLength(4);
+    expect(capture.turns.map((turn) => turn.role)).toEqual([
+      'user',
+      'assistant',
+      'user',
+      'assistant',
+    ]);
+    expect(capture.turns[0]).toMatchObject({
+      visibility: 'visible',
+      sourceEvidence: [
+        {
+          sourceUrl: CHATGPT_SHARE_URL,
+          kind: 'chatgpt_linear_conversation',
+        },
+      ],
+    });
+    expect(capture.turns[0].content).toContain('make a loop of that');
+    expect(capture.turns[1].content).toContain(
+      'while true; do sleep 30m && hive-cleanup -f; done'
+    );
+    expect(capture.turns[3].content).toContain(
+      'screen -dmS auto-cleanup bash -c'
+    );
+  });
+
+  it('formats ChatGPT captures as demo_memory', () => {
+    const capture = parseSharedDialog(
+      readCaseStudyFixture('chatgpt-share-6a3825b9.html'),
+      {
+        sourceUrl: CHATGPT_SHARE_URL,
+        captureMethod: 'static_http',
+      }
+    );
+
+    const memory = formatSharedDialogAsDemoMemory(capture, {
+      demoLabel: 'issue-552-chatgpt-share',
+    });
+
+    expect(memory).toContain('demo_memory');
+    expect(memory.match(/\n {2}event "/g)).toHaveLength(4);
+    expect(memory).toContain('role "user"');
+    expect(memory).toContain('role "assistant"');
+    expect(memory).toContain(
+      'conversationId "6a3825b9-8de4-83ee-9c24-52fd1eb38d24"'
+    );
+    expect(memory).toContain('conversationTitle "Infinite loop script"');
+    expect(memory).toContain(`evidence "${CHATGPT_SHARE_URL}"`);
+    expect(memory).toContain(
+      "screen -dmS auto-cleanup bash -c 'while true; do sleep 30m && hive-cleanup -f; done'"
+    );
+  });
+
+  it('formats captures as shared-dialog meta-language and Markdown', () => {
+    const capture = parseSharedDialog(
+      readCaseStudyFixture('chatgpt-share-6a3825b9.html'),
+      {
+        sourceUrl: CHATGPT_SHARE_URL,
+        captureMethod: 'static_http',
+      }
+    );
+
+    const metaLanguage = formatSharedDialogAsMetaLanguage(capture);
+    expect(metaLanguage).toContain('shared_dialog_capture');
+    expect(metaLanguage).toContain('provider "chatgpt"');
+    expect(metaLanguage).toContain(
+      'turn "0c9f0151-b5a1-402f-afc3-6bd34a0d01d2"'
+    );
+
+    const markdown = formatSharedDialogAsMarkdown(capture);
+    expect(markdown).toContain('# Infinite loop script');
+    expect(markdown).toContain('**User**');
+    expect(markdown).toContain('**Assistant**');
+  });
+
+  it('returns a structured Google AI Mode interstitial diagnostic', () => {
+    const capture = parseSharedDialog(
+      readCaseStudyFixture('google-ai-mode-VG0HhpnAXrBkC0QgP.html'),
+      {
+        sourceUrl: GOOGLE_AI_MODE_URL,
+        captureMethod: 'static_http',
+      }
+    );
+
+    expect(capture).toMatchObject({
+      provider: 'google_ai_mode',
+      sourceUrl: GOOGLE_AI_MODE_URL,
+      captureMethod: 'static_http',
+      status: 'unsupported',
+      turns: [],
+      diagnostics: {
+        unsupportedReason: 'provider_challenge_interstitial',
+      },
+    });
+  });
+
+  it('uses browser-rendered DOM when static Google AI Mode capture is blocked', async () => {
+    const googleHtml = readCaseStudyFixture(
+      'google-ai-mode-VG0HhpnAXrBkC0QgP.html'
+    );
+    const renderHtml = jest.fn(async () => googleHtml);
+    const capture = await captureSharedDialog({
+      url: GOOGLE_AI_MODE_URL,
+      capture: 'browser',
+      fetchImpl: async () => ({
+        ok: true,
+        status: 200,
+        text: async () => googleHtml,
+      }),
+      renderHtml,
+      now: () => '2026-06-25T00:00:00.000Z',
+    });
+
+    expect(renderHtml).toHaveBeenCalledWith(GOOGLE_AI_MODE_URL, {
+      engine: 'puppeteer',
+    });
+    expect(capture).toMatchObject({
+      provider: 'google_ai_mode',
+      captureMethod: 'browser',
+      status: 'unsupported',
+      capturedAt: '2026-06-25T00:00:00.000Z',
+      diagnostics: {
+        unsupportedReason: 'provider_challenge_interstitial',
+      },
+    });
+  });
+});

package/tests/unit/stream.test.js

@@ -0,0 +1,85 @@
+import express from 'express';
+import http from 'http';
+import request from 'supertest';
+import { streamHandler } from '../../src/stream.js';
+
+describe('streamHandler', () => {
+  it('streams upstream content with proxy-safe framing', async () => {
+    const body = '<!DOCTYPE html><html><body>Example Domain</body></html>';
+    let upstreamHeaders;
+    const { server, url } = await startUpstreamServer((req, res) => {
+      upstreamHeaders = req.headers;
+      res.writeHead(200, {
+        'Content-Type': 'text/html; charset=utf-8',
+        'Content-Length': Buffer.byteLength(body),
+      });
+      res.end(body);
+    });
+
+    try {
+      const res = await request(testApp())
+        .get('/stream')
+        .query({ url })
+        .expect(200);
+
+      expect(upstreamHeaders['accept-encoding']).toBe('identity');
+      expect(res.headers['content-type']).toBe('text/html; charset=utf-8');
+      expect(res.headers['content-length']).toBe(
+        String(Buffer.byteLength(body))
+      );
+      expect(res.text).toContain('Example Domain');
+    } finally {
+      await closeServer(server);
+    }
+  });
+
+  it('ends the downstream response when upstream closes after body bytes', async () => {
+    const body = '<!DOCTYPE html><html><body>Example Domain</body></html>';
+    const { server, url } = await startUpstreamServer((req, res) => {
+      res.writeHead(200, {
+        'Content-Type': 'text/html; charset=utf-8',
+      });
+      res.write(body);
+      res.socket.end();
+    });
+
+    try {
+      const res = await request(testApp())
+        .get('/stream')
+        .query({ url })
+        .expect(200);
+
+      expect(res.text).toContain('Example Domain');
+    } finally {
+      await closeServer(server);
+    }
+  });
+});
+
+function testApp() {
+  const app = express();
+  app.get('/stream', streamHandler);
+  return app;
+}
+
+async function startUpstreamServer(handler) {
+  const server = http.createServer(handler);
+  await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve));
+  const { port } = server.address();
+  return {
+    server,
+    url: `http://127.0.0.1:${port}`,
+  };
+}
+
+async function closeServer(server) {
+  await new Promise((resolve, reject) => {
+    server.close((error) => {
+      if (error) {
+        reject(error);
+      } else {
+        resolve();
+      }
+    });
+  });
+}