ampcode-connector 0.1.4 → 0.1.6

package/src/tools/web-read.ts

@@ -0,0 +1,336 @@
+/** Local handler for extractWebPageContent — fetches a URL, converts to Markdown, ranks by objective. */
+
+import { convert, JsPreprocessingPreset } from "@kreuzberg/html-to-markdown";
+import { logger } from "../utils/logger.ts";
+
+export interface WebReadParams {
+  url: string;
+  objective?: string;
+  forceRefetch?: boolean;
+}
+
+export type WebReadResult =
+  | { ok: true; result: { excerpts: string[] } | { fullContent: string } }
+  | { ok: false; error: { code: string; message: string } };
+
+interface Section {
+  heading: string;
+  text: string;
+  index: number;
+}
+
+interface ScoredSection {
+  text: string;
+  score: number;
+  index: number;
+}
+
+type FetchOk = { ok: true; body: string; contentType: string };
+type FetchErr = WebReadResult & { ok: false };
+
+const FETCH = {
+  TIMEOUT_MS: 30_000,
+  USER_AGENT: "Mozilla/5.0 (compatible; AmpBot/1.0)",
+} as const;
+
+const CACHE = {
+  MAX_ENTRIES: 50,
+  TTL_MS: 5 * 60 * 1000,
+} as const;
+
+const RANKING = {
+  MAX_SECTIONS: 10,
+  MAX_SECTION_WORDS: 500,
+  MIN_KEYWORD_LEN: 3,
+  HEADING_BOOST: 2,
+  BIGRAM_BOOST: 1.5,
+  POSITION_DECAY: 0.1,
+  BM25_K1: 1.5,
+  BM25_B: 0.75,
+} as const;
+
+const CLIPPING = {
+  MAX_BYTES: 262_144, // 256 KB — CLI truncation limit
+  MIN_TAIL_BYTES: 100,
+  EXCERPT_SEP_BYTES: 2, // "\n\n" separator
+} as const;
+
+const HTML_OPTIONS = {
+  skipImages: true,
+  preprocessing: { enabled: true, preset: JsPreprocessingPreset.Aggressive },
+};
+
+// biome-ignore format: compact
+const STOP_WORDS = new Set(
+  ("the and for are but not you all can her was one our out " +
+  "has have had been from this that with they which their will " +
+  "each make like just over such than them very some what about " +
+  "into more other then these when where how does also after " +
+  "should would could being there before between those through while using").split(" "),
+);
+
+const encoder = new TextEncoder();
+const decoder = new TextDecoder("utf-8", { fatal: false });
+
+const cache = new Map<string, { markdown: string; createdAt: number }>();
+
+function getCached(url: string): string | undefined {
+  const entry = cache.get(url);
+  if (!entry) return undefined;
+
+  if (Date.now() - entry.createdAt > CACHE.TTL_MS) {
+    cache.delete(url);
+    return undefined;
+  }
+
+  // Re-insert to promote as most-recent (LRU)
+  cache.delete(url);
+  cache.set(url, entry);
+  return entry.markdown;
+}
+
+function setCache(url: string, markdown: string): void {
+  if (cache.size >= CACHE.MAX_ENTRIES) {
+    const oldest = cache.keys().next().value!;
+    cache.delete(oldest);
+  }
+  cache.set(url, { markdown, createdAt: Date.now() });
+}
+
+export async function handleWebRead({ url, objective, forceRefetch }: WebReadParams): Promise<WebReadResult> {
+  let markdown = forceRefetch ? undefined : getCached(url);
+
+  if (!markdown) {
+    const page = await fetchPage(url);
+    if (!page.ok) return page;
+    markdown = convertToMarkdown(page.body, page.contentType);
+    setCache(url, markdown);
+  }
+
+  if (objective) {
+    return { ok: true, result: { excerpts: rankExcerpts(markdown, objective) } };
+  }
+  return { ok: true, result: { fullContent: clipText(markdown) } };
+}
+
+async function fetchPage(url: string): Promise<FetchOk | FetchErr> {
+  let response: Response;
+  try {
+    response = await fetch(url, {
+      signal: AbortSignal.timeout(FETCH.TIMEOUT_MS),
+      redirect: "follow",
+      headers: { "User-Agent": FETCH.USER_AGENT },
+    });
+  } catch (error) {
+    logger.warn("web-read fetch failed", { url, error: String(error) });
+    return fetchError(`Failed to fetch ${url}: ${String(error)}`);
+  }
+
+  if (!response.ok) {
+    return fetchError(`HTTP ${response.status} from ${url}`);
+  }
+
+  const body = await response.text();
+  const contentType = response.headers.get("content-type") ?? "";
+  return { ok: true, body, contentType };
+}
+
+function fetchError(message: string): FetchErr {
+  return { ok: false, error: { code: "fetch-error", message } };
+}
+
+function convertToMarkdown(raw: string, contentType: string): string {
+  if (contentType.includes("text/html") || contentType.includes("application/xhtml")) {
+    return convert(raw, HTML_OPTIONS);
+  }
+
+  if (contentType.includes("application/json")) {
+    try {
+      return `\`\`\`json\n${JSON.stringify(JSON.parse(raw), null, 2)}\n\`\`\``;
+    } catch {
+      return raw;
+    }
+  }
+
+  return raw;
+}
+
+function rankExcerpts(markdown: string, objective: string): string[] {
+  const sections = splitSections(markdown);
+  if (!sections.length) return [clipText(markdown)];
+  const { unigrams, bigrams } = parseTerms(objective);
+  if (!unigrams.length) return [clipText(markdown)];
+  const unigramPatterns = unigrams.map((w) => new RegExp(`\\b${RegExp.escape(w)}\\b`, "g"));
+  const idfWeights = computeIdf(sections, unigramPatterns);
+  const avgDocLen = sections.reduce((sum, s) => sum + (s.text.split(/\s+/).length || 1), 0) / sections.length;
+  const totalSections = sections.length;
+  const scored = sections.map((section) =>
+    scoreSection(section, unigramPatterns, bigrams, idfWeights, avgDocLen, totalSections),
+  );
+  const hits = scored.filter((s) => s.score > 0);
+  if (!hits.length) return [clipText(markdown)];
+  hits.sort((a, b) => b.score - a.score || a.index - b.index);
+  const top = hits.slice(0, RANKING.MAX_SECTIONS);
+  top.sort((a, b) => a.index - b.index);
+  return clipMany(top.map((s) => s.text));
+}
+
+function parseTerms(objective: string): { unigrams: string[]; bigrams: RegExp[] } {
+  const words = objective
+    .toLowerCase()
+    .split(/\W+/)
+    .filter((word) => word.length >= RANKING.MIN_KEYWORD_LEN && !STOP_WORDS.has(word));
+
+  const bigrams = words
+    .slice(0, -1)
+    .map((word, i) => new RegExp(`\\b${RegExp.escape(word)}\\W+${RegExp.escape(words[i + 1]!)}\\b`));
+
+  return { unigrams: words, bigrams };
+}
+
+function computeIdf(sections: Section[], patterns: RegExp[]): number[] {
+  const lowerTexts = sections.map((section) => section.text.toLowerCase());
+  const totalSections = sections.length;
+  return patterns.map((pattern) => {
+    const docFreq = lowerTexts.filter((text) => {
+      pattern.lastIndex = 0;
+      return pattern.test(text);
+    }).length;
+    return docFreq > 0 ? Math.log((totalSections - docFreq + 0.5) / (docFreq + 0.5) + 1) : 0;
+  });
+}
+
+function scoreSection(
+  section: Section,
+  unigramPatterns: RegExp[],
+  bigrams: RegExp[],
+  idfWeights: number[],
+  avgDocLen: number,
+  totalSections: number,
+): ScoredSection {
+  const lowerText = section.text.toLowerCase();
+  const lowerHeading = section.heading.toLowerCase();
+  const docLen = lowerText.split(/\s+/).length || 1;
+
+  // BM25 scoring
+  const { BM25_K1: k1, BM25_B: b } = RANKING;
+  let score = 0;
+  for (let i = 0; i < unigramPatterns.length; i++) {
+    const pattern = unigramPatterns[i]!;
+    pattern.lastIndex = 0;
+    const matches = lowerText.match(pattern);
+    if (matches) {
+      const tf = matches.length;
+      score += idfWeights[i]! * ((tf * (k1 + 1)) / (tf + k1 * (1 - b + b * (docLen / avgDocLen))));
+    }
+  }
+  // Bigram bonus
+  for (const pattern of bigrams) {
+    if (pattern.test(lowerText)) score *= RANKING.BIGRAM_BOOST;
+  }
+
+  // Heading match boost (reuse pre-compiled patterns)
+  if (section.heading) {
+    if (
+      unigramPatterns.some((pattern) => {
+        pattern.lastIndex = 0;
+        return pattern.test(lowerHeading);
+      })
+    ) {
+      score *= RANKING.HEADING_BOOST;
+    }
+  }
+
+  // Position decay — earlier sections get mild boost
+  score *= 1 + RANKING.POSITION_DECAY * (1 - section.index / totalSections);
+  return { text: section.text, score, index: section.index };
+}
+
+function splitSections(markdown: string): Section[] {
+  const raw = parseHeadingSections(markdown);
+  return chunkOversizedSections(raw);
+}
+
+function parseHeadingSections(markdown: string): Section[] {
+  const sections: Section[] = [];
+  let heading = "";
+  let body: string[] = [];
+
+  const flush = () => {
+    const joined = body.join("\n").trim();
+    if (heading || joined) {
+      const text = heading ? `${heading}\n${joined}` : joined;
+      sections.push({ heading, text, index: sections.length });
+    }
+  };
+
+  for (const line of markdown.split("\n")) {
+    if (/^#{1,6}\s/.test(line)) {
+      flush();
+      heading = line;
+      body = [];
+    } else {
+      body.push(line);
+    }
+  }
+  flush();
+
+  return sections;
+}
+
+function chunkOversizedSections(sections: Section[]): Section[] {
+  const result: Section[] = [];
+
+  for (const section of sections) {
+    const wordCount = section.text.split(/\s+/).length;
+    if (wordCount <= RANKING.MAX_SECTION_WORDS) {
+      result.push({ ...section, index: result.length });
+      continue;
+    }
+
+    const paragraphs = section.text.split(/\n{2,}/);
+    let chunk: string[] = [];
+    let chunkWords = 0;
+
+    for (const paragraph of paragraphs) {
+      const paraWords = paragraph.split(/\s+/).length;
+      if (chunkWords + paraWords > RANKING.MAX_SECTION_WORDS && chunk.length > 0) {
+        result.push({ heading: section.heading, text: chunk.join("\n\n"), index: result.length });
+        chunk = [];
+        chunkWords = 0;
+      }
+      chunk.push(paragraph);
+      chunkWords += paraWords;
+    }
+
+    if (chunk.length > 0) {
+      result.push({ heading: section.heading, text: chunk.join("\n\n"), index: result.length });
+    }
+  }
+
+  return result;
+}
+
+function clipText(text: string): string {
+  const bytes = encoder.encode(text);
+  if (bytes.length <= CLIPPING.MAX_BYTES) return text;
+  return decoder.decode(bytes.slice(0, CLIPPING.MAX_BYTES)).replace(/\uFFFD+$/, "");
+}
+
+function clipMany(excerpts: string[]): string[] {
+  let usedBytes = 0;
+  const result: string[] = [];
+
+  for (const excerpt of excerpts) {
+    const excerptBytes = encoder.encode(excerpt).length + CLIPPING.EXCERPT_SEP_BYTES;
+    if (usedBytes + excerptBytes > CLIPPING.MAX_BYTES) {
+      const remaining = CLIPPING.MAX_BYTES - usedBytes;
+      if (remaining > CLIPPING.MIN_TAIL_BYTES) result.push(clipText(excerpt));
+      break;
+    }
+    result.push(excerpt);
+    usedBytes += excerptBytes;
+  }
+
+  return result.length > 0 ? result : [clipText(excerpts[0]!)];
+}

package/src/tools/web-search.ts

@@ -3,6 +3,17 @@
 import Exa from "exa-js";
 import { logger } from "../utils/logger.ts";
 
+let _exa: InstanceType<typeof Exa> | null = null;
+let _exaKey: string | null = null;
+
+function getExa(apiKey: string): InstanceType<typeof Exa> {
+  if (!_exa || _exaKey !== apiKey) {
+    _exa = new Exa(apiKey);
+    _exaKey = apiKey;
+  }
+  return _exa;
+}
+
 export interface SearchParams {
   objective: string;
   searchQueries?: string[];
@@ -15,34 +26,30 @@ interface SearchResultItem {
   excerpts: string[];
 }
 
-type SearchResult =
-  | { ok: true; result: { results: SearchResultItem[]; showParallelAttribution: boolean } }
-  | { ok: false; error: { code: string; message: string } };
+interface SearchResponse {
+  ok: true;
+  result: { results: SearchResultItem[]; showParallelAttribution: boolean };
+}
 
-export async function handleSearch(params: SearchParams, exaApiKey: string): Promise<SearchResult> {
+export async function handleSearch(params: SearchParams, exaApiKey: string): Promise<SearchResponse> {
   const { objective, searchQueries, maxResults = 5 } = params;
   const query = searchQueries?.length ? searchQueries.join(" ") : objective;
 
-  try {
-    const exa = new Exa(exaApiKey);
-    const response = await exa.search(query, {
-      numResults: maxResults,
-      type: "auto",
-      contents: {
-        highlights: { query: objective },
-      },
-    });
-
-    const results: SearchResultItem[] = response.results.map((r) => ({
-      title: r.title ?? "",
-      url: r.url,
-      excerpts: r.highlights?.length ? r.highlights : [],
-    }));
-
-    logger.info(`[SEARCH] Exa returned ${results.length} results for "${query.slice(0, 80)}"`);
-    return { ok: true, result: { results, showParallelAttribution: false } };
-  } catch (err) {
-    logger.error("webSearch2 Exa error", { error: String(err) });
-    return { ok: false, error: { code: "search-error", message: String(err) } };
-  }
+  const exa = getExa(exaApiKey);
+  const response = await exa.search(query, {
+    numResults: maxResults,
+    type: "auto",
+    contents: {
+      highlights: { query: objective },
+    },
+  });
+
+  const results: SearchResultItem[] = response.results.map((r) => ({
+    title: r.title ?? "",
+    url: r.url,
+    excerpts: r.highlights?.length ? r.highlights : [],
+  }));
+
+  logger.info(`[SEARCH] Exa returned ${results.length} results for "${query.slice(0, 80)}"`);
+  return { ok: true, result: { results, showParallelAttribution: false } };
 }

package/src/utils/code-assist.ts

@@ -55,6 +55,6 @@ export function maybeWrap(
   opts: { userAgent: "antigravity" | "pi-coding-agent"; requestIdPrefix: "agent" | "pi"; requestType?: "agent" },
 ): string {
   if (!parsed) return raw;
-  if (parsed["project"]) return raw;
+  if (parsed.project) return raw;
   return wrapRequest({ projectId, model, body: parsed, ...opts });
 }

package/src/utils/logger.ts

@@ -23,6 +23,34 @@ const LOG_LEVELS: Record<LogLevel, number> = {
   error: 3,
 };
 
+const isTTY = !!process.stdout.isTTY;
+
+const RESET = "\x1b[0m";
+const DIM = "\x1b[2m";
+const GREEN = "\x1b[32m";
+const YELLOW = "\x1b[33m";
+const RED = "\x1b[31m";
+
+const LEVEL_COLORS: Record<LogLevel, string> = {
+  debug: DIM,
+  info: "",
+  warn: YELLOW,
+  error: RED,
+};
+
+const ROUTE_COLORS: Record<RouteDecision, string> = {
+  LOCAL_CLAUDE: GREEN,
+  LOCAL_CODEX: GREEN,
+  LOCAL_GEMINI: GREEN,
+  LOCAL_ANTIGRAVITY: GREEN,
+  AMP_UPSTREAM: YELLOW,
+};
+
+function colorize(text: string, color: string): string {
+  if (!isTTY || !color) return text;
+  return `${color}${text}${RESET}`;
+}
+
 let currentLevel: LogLevel = "info";
 
 export function setLogLevel(level: LogLevel): void {
@@ -36,8 +64,9 @@ function shouldLog(level: LogLevel): boolean {
 function format(entry: LogEntry): string {
   const { timestamp, level, message, route, provider, model, duration, error } = entry;
 
-  let line = `${timestamp} [${level.toUpperCase().padEnd(5)}] ${message}`;
-  if (route) line += ` route=${route}`;
+  const tag = colorize(`[${level.toUpperCase().padEnd(5)}]`, LEVEL_COLORS[level]);
+  let line = `${timestamp} ${tag} ${message}`;
+  if (route) line += ` route=${colorize(route, ROUTE_COLORS[route])}`;
   if (provider) line += ` provider=${provider}`;
   if (model) line += ` model=${model}`;
   if (duration !== undefined) line += ` duration=${duration}ms`;

package/src/utils/path.ts

@@ -2,6 +2,11 @@
 
 import { browserPrefixes, passthroughExact, passthroughPrefixes } from "../constants.ts";
 
+const PROVIDER_RE = /^\/api\/provider\/([^/]+)/;
+const SUBPATH_RE = /^\/api\/provider\/[^/]+(\/.*)/;
+const MODEL_RE = /models\/([^/:]+)/;
+const GEMINI_RE = /models\/([^/:]+):(\w+)/;
+
 export function passthrough(pathname: string): boolean {
   if ((passthroughExact as readonly string[]).includes(pathname)) return true;
   return passthroughPrefixes.some((prefix) => pathname.startsWith(prefix));
@@ -13,22 +18,22 @@ export function browser(pathname: string): boolean {
 }
 
 export function provider(pathname: string): string | null {
-  const match = pathname.match(/^\/api\/provider\/([^/]+)/);
+  const match = pathname.match(PROVIDER_RE);
   return match?.[1] ?? null;
 }
 
 export function subpath(pathname: string): string {
-  const match = pathname.match(/^\/api\/provider\/[^/]+(\/.*)/);
+  const match = pathname.match(SUBPATH_RE);
   return match?.[1] ?? pathname;
 }
 
 export function modelFromUrl(url: string): string | null {
-  const match = url.match(/models\/([^/:]+)/);
+  const match = url.match(MODEL_RE);
   return match?.[1] ?? null;
 }
 
 export function gemini(url: string): { model: string; action: string } | null {
-  const match = url.match(/models\/([^/:]+):(\w+)/);
+  const match = url.match(GEMINI_RE);
   if (!match) return null;
   return { model: match[1]!, action: match[2]! };
 }

package/src/utils/stats.ts

@@ -0,0 +1,69 @@
+/** Request statistics tracking with ring buffer. */
+
+import type { RouteDecision } from "../utils/logger.ts";
+
+export interface RequestEntry {
+  timestamp: string;
+  route: RouteDecision;
+  provider: string;
+  model: string;
+  statusCode: number;
+  durationMs: number;
+}
+
+const MAX_ENTRIES = 1000;
+const buffer: RequestEntry[] = [];
+let writeIndex = 0;
+let totalCount = 0;
+const startedAt = Date.now();
+
+export function record(entry: RequestEntry): void {
+  if (buffer.length < MAX_ENTRIES) {
+    buffer.push(entry);
+  } else {
+    buffer[writeIndex] = entry;
+  }
+  writeIndex = (writeIndex + 1) % MAX_ENTRIES;
+  totalCount++;
+}
+
+export interface StatsSnapshot {
+  totalRequests: number;
+  requestsByRoute: Partial<Record<RouteDecision, number>>;
+  count429: number;
+  averageDurationMs: number;
+  uptimeMs: number;
+}
+
+export function snapshot(): StatsSnapshot {
+  const requestsByRoute: Partial<Record<RouteDecision, number>> = {};
+  let count429 = 0;
+  let totalDuration = 0;
+
+  for (const entry of buffer) {
+    requestsByRoute[entry.route] = (requestsByRoute[entry.route] ?? 0) + 1;
+    if (entry.statusCode === 429) count429++;
+    totalDuration += entry.durationMs;
+  }
+
+  return {
+    totalRequests: totalCount,
+    requestsByRoute,
+    count429,
+    averageDurationMs: buffer.length > 0 ? totalDuration / buffer.length : 0,
+    uptimeMs: Date.now() - startedAt,
+  };
+}
+
+export function recentRequests(n: number): RequestEntry[] {
+  const count = Math.min(n, buffer.length);
+  if (count === 0) return [];
+
+  const result: RequestEntry[] = [];
+  let idx = (writeIndex - count + buffer.length) % buffer.length;
+  for (let i = 0; i < count; i++) {
+    result.push(buffer[idx]!);
+    idx = (idx + 1) % buffer.length;
+  }
+  return result;
+}

package/src/utils/streaming.ts

@@ -24,7 +24,7 @@ export function parse(raw: string): Chunk[] {
         chunk.id = line.slice(3).trim();
       } else if (line.startsWith("retry:")) {
         const val = parseInt(line.slice(6).trim(), 10);
-        if (!isNaN(val)) chunk.retry = val;
+        if (!Number.isNaN(val)) chunk.retry = val;
       }
     }
 
@@ -48,12 +48,13 @@ export function encode(chunk: Chunk): string {
   return result;
 }
 
+const decoder = new TextDecoder();
+const encoder = new TextEncoder();
+
 export function transform(
   source: ReadableStream<Uint8Array>,
   fn: (data: string) => string,
 ): ReadableStream<Uint8Array> {
-  const decoder = new TextDecoder();
-  const encoder = new TextEncoder();
   let buffer = "";
 
   const stream = new TransformStream<Uint8Array, Uint8Array>({
@@ -86,13 +87,11 @@ export function transform(
   return source.pipeThrough(stream);
 }
 
-export function headers(): Record<string, string> {
-  return {
-    "Content-Type": "text/event-stream",
-    "Cache-Control": "no-cache",
-    Connection: "keep-alive",
-  };
-}
+const SSE_HEADERS: Readonly<Record<string, string>> = {
+  "Content-Type": "text/event-stream",
+  "Cache-Control": "no-cache",
+  Connection: "keep-alive",
+};
 
 const forwardedHeaders = [
   "x-request-id",
@@ -114,7 +113,7 @@ export function proxy(upstream: Response, rewrite?: (data: string) => string): R
 
   const body = rewrite ? transform(upstream.body, rewrite) : upstream.body;
 
-  const h: Record<string, string> = { ...headers() };
+  const h: Record<string, string> = { ...SSE_HEADERS };
   for (const name of forwardedHeaders) {
     const value = upstream.headers.get(name);
     if (value) h[name] = value;

package/tsconfig.json

@@ -11,7 +11,7 @@
     // Bundler mode
     "moduleResolution": "bundler",
     "allowImportingTsExtensions": true,
-    "verbatimModuleSyntax": true,
+    "verbatimModuleSyntax": false,
     "noEmit": true,
 
     // Best practices
@@ -22,8 +22,8 @@
     "noImplicitOverride": true,
 
     // Some stricter flags (disabled by default)
-    "noUnusedLocals": false,
-    "noUnusedParameters": false,
+    "noUnusedLocals": true,
+    "noUnusedParameters": true,
     "noPropertyAccessFromIndexSignature": false
   },
   "exclude": ["references"]