@link-assistant/web-capture 1.10.8 → 1.10.10

package/CHANGELOG.md

@@ -1,5 +1,17 @@
 # @link-assistant/web-capture
 
+## 1.10.10
+
+### Patch Changes
+
+- 65d7dcc: Add shared AI dialog capture for ChatGPT share links with structured transcript output and unsupported-provider diagnostics.
+
+## 1.10.9
+
+### Patch Changes
+
+- Auto-release unreleased changes
+
 ## 1.10.8
 
 ### Patch Changes

package/README.md

@@ -15,6 +15,7 @@ A CLI and microservice to fetch URLs and render them as:
 - **ZIP archive**: Markdown + locally downloaded images
 - **PDF**: Print-quality document with embedded images
 - **DOCX**: Word document with embedded images
+- **Shared AI dialog transcript**: ChatGPT share captures with structured diagnostics for unsupported providers
 
 ## Installation
 
@@ -99,6 +100,10 @@ web-capture https://example.com --no-extract-latex --no-post-process -o page.md
 # Structured search-provider capture (JSON by default)
 web-capture search "formal methods" --provider wikipedia
 
+# Shared AI dialog capture (JSON by default)
+web-capture shared-dialog https://chatgpt.com/share/SHARE_ID
+web-capture shared-dialog https://chatgpt.com/share/SHARE_ID --format demo-memory -o -
+
 # Start as API server
 web-capture --serve
 
@@ -321,6 +326,17 @@ Example response (`format=json`):
 }
 ```
 
+### Shared Dialog Endpoint
+
+```
+GET /shared-dialog?url=<URL>&format=json|meta-language|demo-memory|markdown|txt
+```
+
+Captures replayable shared AI dialog transcripts. ChatGPT share links are
+decoded from their embedded `linear_conversation` data. Providers that cannot
+currently expose transcript data, such as Google AI Mode interstitial pages,
+return a structured unsupported diagnostic.
+
 ## CLI Reference
 
 ### Server Mode

package/bin/web-capture.js

@@ -1,8 +1,4 @@
 #!/usr/bin/env node
-// CLI entry point for web-capture
-// Supports two modes:
-// 1. Server mode: web-capture --serve [--port 3000]
-// 2. Capture mode: web-capture <url> [options]
 
 import fs from 'fs';
 import path from 'path';
@@ -27,7 +23,11 @@ function makeVerboseLog(enabled) {
   });
 }
 
-// Create configuration using lino-arguments pattern
+const originalConsoleLog = console.log;
+console.log = (...args) =>
+  String(args[0]).includes(' variables from ')
+    ? console.error(...args)
+    : originalConsoleLog(...args);
 const config = makeConfig({
   yargs: ({ yargs, getenv }) =>
     yargs
@@ -49,6 +49,15 @@ const config = makeConfig({
             description: 'Search query',
           })
       )
+      .command(
+        'shared-dialog <sharedDialogUrl>',
+        'Capture replayable shared AI dialog transcripts',
+        (yargs) =>
+          yargs.positional('sharedDialogUrl', {
+            type: 'string',
+            description: 'Shared AI dialog URL',
+          })
+      )
       .option('provider', {
         type: 'string',
         description:
@@ -78,7 +87,7 @@ const config = makeConfig({
         alias: 'f',
         type: 'string',
         description:
-          'Output format: markdown, md, html, txt, text, image, png, jpeg, pdf, docx, archive',
+          'Output format: markdown, md, html, txt, text, image, png, jpeg, pdf, docx, archive, json, meta-language, demo-memory',
         default: 'markdown',
       })
       .option('theme', {
@@ -289,7 +298,8 @@ const config = makeConfig({
         'Capture private Google Doc with API token'
       )
       .epilogue(
-        'API Endpoints (in server mode):\n  GET /html?url=<URL>&engine=<ENGINE>           Get rendered HTML\n  GET /markdown?url=<URL>                       Get Markdown conversion\n  GET /txt?url=<URL>                            Get text content\n  GET /image?url=<URL>&format=png|jpeg&theme=light|dark  Screenshot\n  GET /archive?url=<URL>&localImages=true&documentFormat=markdown|html  ZIP archive\n  GET /pdf?url=<URL>&theme=light|dark           PDF with embedded images\n  GET /docx?url=<URL>                           DOCX with embedded images\n  GET /fetch?url=<URL>                          Proxy fetch\n  GET /stream?url=<URL>                         Streaming proxy'
+        'API Endpoints (in server mode):\n  GET /html?url=<URL>&engine=<ENGINE>           Get rendered HTML\n  GET /markdown?url=<URL>                       Get Markdown conversion\n  GET /txt?url=<URL>                            Get text content\n  GET /image?url=<URL>&format=png|jpeg&theme=light|dark  Screenshot\n  GET /archive?url=<URL>&localImages=true&documentFormat=markdown|html  ZIP archive\n  GET /pdf?url=<URL>&theme=light|dark           PDF with embedded images\n  GET /docx?url=<URL>                           DOCX with embedded images\n  GET /fetch?url=<URL>                          Proxy fetch\n  GET /stream?url=<URL>                         Streaming proxy' +
+          '\n  GET /shared-dialog?url=<URL>&format=json|meta-language|demo-memory|markdown|txt  Shared AI dialog capture'
       )
       .strict(),
   lenv: {
@@ -297,6 +307,7 @@ const config = makeConfig({
     path: '.lenv',
   },
 });
+console.log = originalConsoleLog;
 
 function getUrlArgument() {
   if (typeof config.url === 'string' && config.url.length > 0) {
@@ -331,6 +342,7 @@ async function startServer(port) {
       console.log(`  GET /docx?url=<URL>       - DOCX with embedded images`);
       console.log(`  GET /fetch?url=<URL>      - Proxy fetch content`);
       console.log(`  GET /stream?url=<URL>     - Stream content`);
+      console.log(`  GET /shared-dialog?url=<URL> - Shared AI dialog capture`);
       console.log('');
       console.log('Press Ctrl+C to stop the server');
       resolve(server);
@@ -1370,14 +1382,8 @@ async function captureUrl(url, options) {
 
 async function runSearch(query) {
   const { search, formatSearchAsMarkdown } = await import('../src/search.js');
-  // `--format` defaults to `markdown` for capture mode, but search results are
-  // primarily structured JSON. Only honor the format flag when the user passed
-  // it explicitly; otherwise default search output to JSON.
-  const formatPassedExplicitly = process.argv.some(
-    (arg) => arg === '-f' || arg === '--format' || arg.startsWith('--format=')
-  );
   const format = (
-    formatPassedExplicitly ? config.format : 'json'
+    formatPassedExplicitly() ? config.format : 'json'
   ).toLowerCase();
   const result = await search({
     query,
@@ -1391,13 +1397,40 @@ async function runSearch(query) {
   }
 }
 
+async function runSharedDialog(sharedDialogUrl) {
+  const { runSharedDialogCli } = await import('../src/shared-dialog-cli.js');
+  await runSharedDialogCli(sharedDialogUrl, {
+    capture: config.capture,
+    engine: config.engine,
+    format: formatPassedExplicitly() ? config.format : 'json',
+    output: config.output,
+  });
+}
+
+function formatPassedExplicitly() {
+  return process.argv.some(
+    (arg) => arg === '-f' || arg === '--format' || arg.startsWith('--format=')
+  );
+}
+
 async function main() {
   const url = getUrlArgument();
-  // The `query` key is only present when the `search <query>` command matched.
   const isSearch = Object.prototype.hasOwnProperty.call(config, 'query');
+  const isSharedDialog = Object.prototype.hasOwnProperty.call(
+    config,
+    'sharedDialogUrl'
+  );
 
-  if (isSearch) {
-    // Search mode
+  if (isSharedDialog) {
+    const sharedDialogUrl =
+      typeof config.sharedDialogUrl === 'string' ? config.sharedDialogUrl : '';
+    if (!sharedDialogUrl.trim()) {
+      console.error('Error: Missing shared-dialog URL');
+      console.error('Usage: web-capture shared-dialog <url> [--format <fmt>]');
+      process.exit(1);
+    }
+    await runSharedDialog(sharedDialogUrl);
+  } else if (isSearch) {
     const query = typeof config.query === 'string' ? config.query : '';
     if (!query.trim()) {
       console.error('Error: Missing search query');
@@ -1406,7 +1439,6 @@ async function main() {
     }
     await runSearch(query);
   } else if (config.serve) {
-    // Server mode
     await startServer(config.port);
   } else if (url) {
     // --no-extract-images is an alias for --embed-images
@@ -1429,7 +1461,6 @@ async function main() {
       config.archiveFormat = archiveFormat === 'gz' ? 'tar.gz' : archiveFormat;
       config.format = 'archive';
     }
-    // Capture mode
     await captureUrl(url, {
       format: config.format,
       output: config.output,
@@ -1457,7 +1488,6 @@ async function main() {
       capture: config.capture,
     });
   } else {
-    // No arguments - show error
     console.error('Error: Missing URL or --serve flag');
     console.error('Run with --help for usage information');
     process.exit(1);

package/docker-compose.yml

@@ -5,5 +5,7 @@ services:
       dockerfile: Dockerfile
     ports:
       - '3000:3000'
+    extra_hosts:
+      - 'host.docker.internal:host-gateway'
     environment:
       - PORT=3000

package/jest.config.mjs

@@ -9,6 +9,7 @@ export default {
   testEnvironment: 'node',
   verbose: true,
   testMatch: [
+    '**/tests/ci/**/*.test.js',
     '**/tests/unit/**/*.test.js',
     '**/tests/mock/**/*.test.js',
     '**/tests/e2e/**/*.test.js',

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@link-assistant/web-capture",
-  "version": "1.10.8",
+  "version": "1.10.10",
   "description": "CLI and microservice to render web pages as HTML, Markdown, or PNG",
   "main": "src/index.js",
   "bin": {

package/src/fetch.js

@@ -1,5 +1,6 @@
 import fetch from 'node-fetch';
 import { convertGoogleDriveUrl } from './lib.js';
+import { copyProxyResponseHeaders } from './proxy-headers.js';
 
 export async function fetchHandler(req, res) {
   const url = req.query.url;
@@ -10,21 +11,7 @@ export async function fetchHandler(req, res) {
     const response = await fetch(convertGoogleDriveUrl(url));
     // Copy status and headers
     res.status(response.status);
-
-    // Set default content type if not present
-    const contentType = response.headers.get('content-type') || 'text/plain';
-    res.setHeader('Content-Type', contentType);
-
-    // Copy other headers
-    for (const [key, value] of response.headers.entries()) {
-      if (
-        key.toLowerCase() !== 'transfer-encoding' &&
-        key.toLowerCase() !== 'content-encoding' &&
-        key.toLowerCase() !== 'content-length'
-      ) {
-        res.setHeader(key, value);
-      }
-    }
+    copyProxyResponseHeaders(response.headers, res);
 
     // Get the response body as buffer and send it
     const buffer = await response.buffer();

package/src/gdocs.js

@@ -42,6 +42,12 @@ const GDOCS_EXPORT_BASE = 'https://docs.google.com/document/d';
 const GDOCS_API_BASE = 'https://docs.googleapis.com/v1/documents';
 const DEFAULT_USER_AGENT =
   'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36';
+const PUBLIC_EXPORT_HEADERS = {
+  'User-Agent': DEFAULT_USER_AGENT,
+  'Accept-Charset': 'utf-8',
+  'Accept-Language': 'en-US,en;q=0.9',
+  'Accept-Encoding': 'identity',
+};
 
 /**
  * Supported Google Docs export formats.
@@ -165,11 +171,7 @@ export async function fetchGoogleDoc(url, options = {}) {
     hasApiToken: Boolean(apiToken),
   }));
 
-  const headers = {
-    'User-Agent': DEFAULT_USER_AGENT,
-    'Accept-Charset': 'utf-8',
-    'Accept-Language': 'en-US,en;q=0.9',
-  };
+  const headers = { ...PUBLIC_EXPORT_HEADERS };
 
   if (apiToken) {
     headers.Authorization = `Bearer ${apiToken}`;

package/src/github.js

@@ -244,6 +244,7 @@ async function fetchGithubText(url, options = {}) {
 function githubHeaders(accept) {
   const headers = {
     Accept: accept,
+    'Accept-Encoding': 'identity',
     'User-Agent': GITHUB_USER_AGENT,
     'X-GitHub-Api-Version': '2022-11-28',
   };

package/src/index.js

@@ -13,6 +13,7 @@ import { animationHandler } from './animation.js';
 import { figuresHandler } from './figures.js';
 import { themedImageHandler } from './themed-image.js';
 import { searchHandler } from './search.js';
+import { sharedDialogHandler } from './shared-dialog.js';
 
 const app = express();
 const port = process.env.PORT || 3000;
@@ -31,6 +32,7 @@ app.get('/animation', animationHandler);
 app.get('/figures', figuresHandler);
 app.get('/themed-image', themedImageHandler);
 app.get('/search', searchHandler);
+app.get('/shared-dialog', sharedDialogHandler);
 
 // Start the server if this is the main module
 const isMainModule =
@@ -78,3 +80,13 @@ export {
   formatSearchAsMarkdown,
   SEARCH_PROVIDERS,
 } from './search.js';
+export {
+  captureSharedDialog,
+  formatSharedDialogAsDemoMemory,
+  formatSharedDialogAsMarkdown,
+  formatSharedDialogAsMetaLanguage,
+  formatSharedDialogAsText,
+  formatSharedDialogResult,
+  parseSharedDialog,
+  sharedDialogHandler,
+} from './shared-dialog.js';

package/src/lib.js

@@ -15,6 +15,9 @@ import { retry } from './retry.js';
 const STACKPRINTER_RETRIES = 3;
 const STACKPRINTER_RETRY_BASE_DELAY_MS = 1000;
 const GOOGLE_DRIVE_FILE_ID_RE = /^[A-Za-z0-9_-]+$/;
+const HTML_FETCH_HEADERS = {
+  'accept-encoding': 'identity',
+};
 
 export async function fetchHtml(url) {
   if (!url) {
@@ -30,7 +33,7 @@ export async function fetchHtml(url) {
     return await fetchStackPrinterHtml(stackPrinterUrl);
   }
 
-  const response = await fetch(url);
+  const response = await fetchHtmlResponse(url);
   return response.text();
 }
 
@@ -194,7 +197,7 @@ function imageExtensionForContentType(contentType) {
 async function fetchStackPrinterHtml(url) {
   return await retry(
     async () => {
-      const response = await fetch(url);
+      const response = await fetchHtmlResponse(url);
       if (!response.ok) {
         throw new Error(`StackPrinter returned HTTP ${response.status}`);
       }
@@ -212,6 +215,12 @@ async function fetchStackPrinterHtml(url) {
   );
 }
 
+function fetchHtmlResponse(url) {
+  return fetch(url, {
+    headers: HTML_FETCH_HEADERS,
+  });
+}
+
 function isStackPrinterTransientError(html) {
   return /Ooooops|Please try again later/i.test(html);
 }

package/src/proxy-headers.js

@@ -0,0 +1,60 @@
+const HOP_BY_HOP_HEADERS = new Set([
+  'connection',
+  'keep-alive',
+  'proxy-authenticate',
+  'proxy-authorization',
+  'te',
+  'trailer',
+  'transfer-encoding',
+  'upgrade',
+]);
+
+const TRANSFORMED_BODY_HEADERS = new Set(['content-encoding']);
+
+export function getProxyHeaderEntries(sourceHeaders, options = {}) {
+  const { preserveContentLength = false } = options;
+  const connectionTokens = getConnectionTokens(sourceHeaders);
+  const isEncoded = Boolean(sourceHeaders.get('content-encoding'));
+  const entries = [];
+
+  for (const [key, value] of sourceHeaders.entries()) {
+    const lowerKey = key.toLowerCase();
+    if (
+      HOP_BY_HOP_HEADERS.has(lowerKey) ||
+      TRANSFORMED_BODY_HEADERS.has(lowerKey) ||
+      connectionTokens.has(lowerKey)
+    ) {
+      continue;
+    }
+    if (
+      lowerKey === 'content-length' &&
+      (!preserveContentLength || isEncoded)
+    ) {
+      continue;
+    }
+    entries.push([key, value]);
+  }
+
+  return entries;
+}
+
+export function copyProxyResponseHeaders(sourceHeaders, res, options = {}) {
+  const contentType = sourceHeaders.get('content-type') || 'text/plain';
+  res.setHeader('Content-Type', contentType);
+
+  for (const [key, value] of getProxyHeaderEntries(sourceHeaders, options)) {
+    if (key.toLowerCase() !== 'content-type') {
+      res.setHeader(key, value);
+    }
+  }
+}
+
+function getConnectionTokens(sourceHeaders) {
+  const connection = sourceHeaders.get('connection') || '';
+  return new Set(
+    connection
+      .split(',')
+      .map((token) => token.trim().toLowerCase())
+      .filter(Boolean)
+  );
+}

package/src/shared-dialog-cli.js

@@ -0,0 +1,25 @@
+import fs from 'fs';
+import path from 'path';
+
+import {
+  captureSharedDialog,
+  formatSharedDialogResult,
+} from './shared-dialog.js';
+
+export async function runSharedDialogCli(sharedDialogUrl, options = {}) {
+  const format = (options.format || 'json').toLowerCase();
+  const capture = await captureSharedDialog({
+    url: sharedDialogUrl,
+    capture: options.capture,
+    engine: options.engine,
+  });
+  const body = formatSharedDialogResult(capture, format);
+
+  if (options.output && options.output !== '-') {
+    fs.mkdirSync(path.dirname(options.output), { recursive: true });
+    fs.writeFileSync(options.output, body, 'utf-8');
+    console.error(`Shared dialog (${format}) saved to: ${options.output}`);
+  } else {
+    process.stdout.write(body);
+  }
+}