@maycombhq/atticus 2.0.0-beta.2 → 2.0.0-beta.22

package/dist/{chunk-7EKXJZ2Y.js → chunk-6ZGLAACW.js}

@@ -5,13 +5,16 @@ var __atticusUrl = (p) => new URL("./" + p, import.meta.url).href;
 var __atticusPath = (p) => __atticusFileURLToPath(__atticusUrl(p));
 import {
   parseXml
-} from "./chunk-MC3H7736.js";
+} from "./chunk-CQDGOPGH.js";
 import {
+  JOINED_BY_PERSON,
   ZipError,
   peekZip,
-  readZip,
+  readZip
+} from "./chunk-Q2TECDIT.js";
+import {
   require_dist
-} from "./chunk-S2A7YC3Q.js";
+} from "./chunk-BQGDB6ZL.js";
 import {
   __toESM
 } from "./chunk-TV45EUGM.js";
@@ -92,9 +95,9 @@ function isoDate(value) {
   return !Number.isNaN(d.getTime()) && d.toISOString().startsWith(m[0]) ? m[0] : void 0;
 }
 function sourceDocument(fileName, content, docType, source, meta = {}) {
-  const clean = (s) => s && s.trim() ? s.trim() : void 0;
-  const version = clean(meta.version);
-  const author = clean(meta.author);
+  const clean2 = (s) => s && s.trim() ? s.trim() : void 0;
+  const version = clean2(meta.version);
+  const author = clean2(meta.author);
   return {
     schemaVersion: 1,
     kind: "SourceDocument",
@@ -630,7 +633,7 @@ function reconcile(input) {
     add({
       key: i.id,
       type: "code-only",
-      summary: `In code ${i.from} ${what} ${i.to}${i.sync === "async" ? "" : ` (${i.method ?? ""} ${i.normalizedPath}`.trimEnd() + ")"}${i.confidence === "inferred" ? ", seen at one end only" : ""}; the diagram doesn't show it.`,
+      summary: `In code ${i.from} ${what} ${i.to}${i.sync === "async" ? "" : ` (${i.method ?? ""} ${i.normalizedPath}`.trimEnd() + ")"}${i.confidence === "inferred" && !i.flags?.includes(JOINED_BY_PERSON) ? ", seen at one end only" : ""}; the diagram doesn't show it.`,
       records: [i.id],
       suggestion: {
         text: `Add it to the diagram, or record why it is left out`,
@@ -1739,7 +1742,7 @@ function readHtml(src) {
   }
   if (!meta.date)
     delete meta.date;
-  const clean = src.replace(/<!--[\s\S]*?-->/g, blank).replace(/<!\[CDATA\[[\s\S]*?\]\]>/g, blank).replace(/<![^>]*>|<\?[\s\S]*?\?>/g, blank).replace(/<(script|style|noscript|template|svg)\b[\s\S]*?<\/\1\s*>/gi, blank).replace(/<head\b[\s\S]*?<\/head\s*>/gi, blank);
+  const clean2 = src.replace(/<!--[\s\S]*?-->/g, blank).replace(/<!\[CDATA\[[\s\S]*?\]\]>/g, blank).replace(/<![^>]*>|<\?[\s\S]*?\?>/g, blank).replace(/<(script|style|noscript|template|svg)\b[\s\S]*?<\/\1\s*>/gi, blank).replace(/<head\b[\s\S]*?<\/head\s*>/gi, blank);
   const s = new Sections();
   const tables = [];
   let line = "";
@@ -1752,7 +1755,7 @@ function readHtml(src) {
   let lineNo = 1;
   const lineAt = (to) => {
     for (; pos < to; pos++)
-      if (clean.charCodeAt(pos) === 10)
+      if (clean2.charCodeAt(pos) === 10)
         lineNo++;
     return lineNo;
   };
@@ -1779,8 +1782,8 @@ function readHtml(src) {
       line += t;
     }
   };
-  for (let m = tag.exec(clean); m; m = tag.exec(clean)) {
-    text(clean.slice(last, m.index), last);
+  for (let m = tag.exec(clean2); m; m = tag.exec(clean2)) {
+    text(clean2.slice(last, m.index), last);
     last = tag.lastIndex;
     const close = m[1] === "/";
     const name = m[2].toLowerCase();
@@ -1832,7 +1835,7 @@ function readHtml(src) {
     else if (BLOCK.has(name) && !heading && cell === void 0)
       flush();
   }
-  text(clean.slice(last), last);
+  text(clean2.slice(last), last);
   flush();
   return {
     ...title ? { title } : {},
@@ -1884,7 +1887,11 @@ function wordText(p) {
         out += "\n";
       else if (c.name === "noBreakHyphen")
         out += "-";
-      else if (c.name !== "pPr" && c.name !== "rPr" && c.name !== "del")
+      else if (c.name === "AlternateContent") {
+        const one = elements(c).find((x) => x.name === "Choice") ?? elements(c).find((x) => x.name === "Fallback");
+        if (one)
+          walk(one);
+      } else if (c.name !== "pPr" && c.name !== "rPr" && c.name !== "del" && c.name !== "moveFrom")
         walk(c);
     }
   };
@@ -2036,29 +2043,60 @@ var colIndex = (ref2) => {
   return n - 1;
 };
 var stringItem = (si) => elements(si).flatMap((c) => c.name === "t" ? [c] : c.name === "r" ? childrenNamed(c, "t") : []).map(textOf).join("");
-function readXlsx(zip) {
+function readXlsx(zip, maxCells = MAX_CELLS) {
   const wb = need(zip, "xl/workbook.xml");
   const rels = relationships(zip.text("xl/_rels/workbook.xml.rels"));
   const shared = childrenNamed(parseXmlTree(zip.text("xl/sharedStrings.xml") ?? ""), "si").map(stringItem);
   const { title, meta } = coreProps(zip);
   const s = new Sections();
   const tables = [];
+  const notes = [];
+  let budget = maxCells;
+  let unread = false;
   findAll(child(wb, "sheets"), "sheet").forEach((sheet, i) => {
+    const name = sheet.attrs.name ?? `Sheet ${i + 1}`;
     const target = rels.get(relId(sheet) ?? "");
     const part = target ? resolvePart("xl", target) : void 0;
     const doc = part ? parseXmlTree(zip.text(part) ?? "") : void 0;
     const rows = [];
     let prev;
+    let squeezed = false;
     for (const row of findAll(child(doc, "sheetData"), "row")) {
+      if (budget <= 0) {
+        if (childrenNamed(row, "c").length)
+          unread = true;
+        if (unread)
+          break;
+        continue;
+      }
       const r = Number(row.attrs.r) || (prev ?? 0) + 1;
+      if (r > EXCEL_ROWS) {
+        notes.push(`${name}: row ${r} is past Excel's last row; skipped.`);
+        continue;
+      }
       if (prev !== void 0)
-        for (let k = prev + 1; k < r; k++)
+        for (let k = prev + 1; k < Math.min(r, prev + 1 + MAX_GAP); k++)
           rows.push([]);
       prev = r;
       const cells2 = [];
       let next = 0;
+      let shift = 0;
       for (const c of childrenNamed(row, "c")) {
-        const at = c.attrs.r ? colIndex(c.attrs.r) : next;
+        const col = c.attrs.r ? colIndex(c.attrs.r) : next;
+        if (col >= EXCEL_COLUMNS)
+          continue;
+        if (budget <= 0) {
+          unread = true;
+          break;
+        }
+        budget--;
+        next = col + 1;
+        let at = Math.max(0, col - shift);
+        if (at - cells2.length > MAX_GAP) {
+          shift += at - cells2.length - MAX_GAP;
+          at = cells2.length + MAX_GAP;
+          squeezed = true;
+        }
         const v = textOf(child(c, "v"));
         let value;
         switch (c.attrs.t) {
@@ -2077,31 +2115,61 @@ function readXlsx(zip) {
         while (cells2.length < at)
           cells2.push("");
         cells2[at] = value;
-        next = at + 1;
       }
       while (cells2.length && cells2.at(-1) === "")
         cells2.pop();
       rows.push(cells2);
     }
+    if (squeezed)
+      notes.push(`${name}: more than ${MAX_GAP} blank columns between cells were left out.`);
     tables.push(rows);
-    s.heading(sheet.attrs.name ?? `Sheet ${i + 1}`, 1, { page: i + 1 });
+    s.heading(name, 1, { page: i + 1 });
     for (const r of rows)
       s.add(r.join("	"));
   });
-  return { ...title ? { title } : {}, sections: s.done(), tables, meta, notes: [] };
+  if (unread)
+    notes.push(`The workbook has more than ${maxCells} cells; the rest wasn't read.`);
+  return { ...title ? { title } : {}, sections: s.done(), tables, meta, notes };
 }
+var EXCEL_ROWS = 1048576;
+var EXCEL_COLUMNS = 16384;
+var MAX_GAP = 1e3;
+var MAX_CELLS = 2e6;
 
 // packages/ingestors/dist/pdf.js
 import { constants, inflateSync } from "node:zlib";
 var NO_TEXT = "No text was found in this PDF. It may be a scanned image, so a person needs to read it.";
 var MAX_STREAM = 64 * 1024 * 1024;
+var MAX_DECODED = 256 * 1024 * 1024;
+var MAX_PACKED = 1e5;
+var MAX_PAGES = 1e4;
+var MAX_TEXT = 64 * 1024 * 1024;
 var refs = (s) => [...s.matchAll(/(\d+)\s+\d+\s+R\b/g)].map((m) => Number(m[1]));
 var ref = (dict, key) => {
   const m = new RegExp(`/${key}\\s+(\\d+)\\s+\\d+\\s+R\\b`).exec(dict);
   return m ? Number(m[1]) : void 0;
 };
 var isType = (dict, type) => new RegExp(`/Type\\s*/${type}(?![A-Za-z])`).test(dict);
-function decodeStream(o) {
+var tooMuch = () => new ReadError(`This PDF's streams inflate to more than ${MAX_DECODED / 1024 / 1024} MB, so it isn't read`);
+function decodeStream(o, d) {
+  if (d.cache.has(o))
+    return d.cache.get(o);
+  if (o.stream && d.left <= 0)
+    throw tooMuch();
+  const max = Math.min(MAX_STREAM, d.left);
+  const out = decodeOnce(o, max);
+  if (out === "too-big") {
+    if (max < MAX_STREAM)
+      throw tooMuch();
+    d.cache.set(o, void 0);
+    return void 0;
+  }
+  d.cache.set(o, out);
+  if (out)
+    d.left -= out.length;
+  return out;
+}
+function decodeOnce(o, max) {
   if (!o.stream)
     return void 0;
   const filter = /\/Filter\s*(\[[^\]]*\]|\/\w+)/.exec(o.dict)?.[1];
@@ -2110,20 +2178,23 @@ function decodeStream(o) {
   const names = filter.match(/\/\w+/g) ?? [];
   if (names.length !== 1 || names[0] !== "/FlateDecode")
     return void 0;
+  const big = (e) => e.code === "ERR_BUFFER_TOO_LARGE";
   try {
-    return inflateSync(o.stream, { maxOutputLength: MAX_STREAM });
-  } catch {
+    return inflateSync(o.stream, { maxOutputLength: max });
+  } catch (e) {
+    if (big(e))
+      return "too-big";
     try {
       return inflateSync(o.stream, {
         finishFlush: constants.Z_SYNC_FLUSH,
-        maxOutputLength: MAX_STREAM
+        maxOutputLength: max
       });
-    } catch {
-      return void 0;
+    } catch (e2) {
+      return big(e2) ? "too-big" : void 0;
     }
   }
 }
-function objects(src, bytes) {
+function objects(src, bytes, d) {
   const out = /* @__PURE__ */ new Map();
   const re = /(\d+)\s+(\d+)\s+obj\b/g;
   for (let m = re.exec(src); m; m = re.exec(src)) {
@@ -2156,12 +2227,14 @@ function objects(src, bytes) {
   for (const o of [...out.values()]) {
     if (!isType(o.dict, "ObjStm"))
       continue;
-    const data = decodeStream(o)?.toString("latin1");
-    const n = Number(/\/N\s+(\d+)/.exec(o.dict)?.[1] ?? 0);
+    const data = decodeStream(o, d)?.toString("latin1");
     const first = Number(/\/First\s+(\d+)/.exec(o.dict)?.[1] ?? 0);
-    if (!data || !n)
+    if (!data)
       continue;
     const head = data.slice(0, first).trim().split(/\s+/).map(Number);
+    const n = Math.min(Number(/\/N\s+(\d+)/.exec(o.dict)?.[1] ?? 0), Math.floor(head.length / 2), MAX_PACKED);
+    if (!n)
+      continue;
     for (let k = 0; k < n; k++) {
       const num = head[2 * k];
       const off = first + head[2 * k + 1];
@@ -2230,28 +2303,86 @@ function readLiteral(src, i) {
   return [src.slice(i, k), k + 1];
 }
 function hexBytes(h) {
-  const clean = h.replace(/[^0-9a-fA-F]/g, "");
-  const even = clean.length % 2 ? clean + "0" : clean;
+  const clean2 = h.replace(/[^0-9a-fA-F]/g, "");
+  const even = clean2.length % 2 ? clean2 + "0" : clean2;
   const out = [];
   for (let i = 0; i < even.length; i += 2)
     out.push(parseInt(even.slice(i, i + 2), 16));
   return out;
 }
-function contentText(src) {
+function parseToUnicode(cmap) {
+  const map = /* @__PURE__ */ new Map();
+  const utf16 = (h) => {
+    const b = hexBytes(h);
+    let out = "";
+    for (let i = 0; i + 1 < b.length; i += 2)
+      out += String.fromCharCode(b[i] << 8 | b[i + 1]);
+    return out;
+  };
+  const space = /begincodespacerange\s*<([0-9a-fA-F]+)>/.exec(cmap)?.[1];
+  const bytes = space && space.length <= 2 ? 1 : 2;
+  for (const block of cmap.matchAll(/beginbfchar([\s\S]*?)endbfchar/g))
+    for (const m of block[1].matchAll(/<([0-9a-fA-F]+)>\s*<([0-9a-fA-F]*)>/g))
+      map.set(parseInt(m[1], 16), utf16(m[2]));
+  for (const block of cmap.matchAll(/beginbfrange([\s\S]*?)endbfrange/g))
+    for (const m of block[1].matchAll(/<([0-9a-fA-F]+)>\s*<([0-9a-fA-F]+)>\s*(?:<([0-9a-fA-F]*)>|\[([^\]]*)\])/g)) {
+      const lo = parseInt(m[1], 16);
+      const hi = Math.min(parseInt(m[2], 16), lo + 65535);
+      if (m[3] !== void 0) {
+        const start = utf16(m[3]);
+        const last = start.charCodeAt(start.length - 1);
+        for (let c = lo; c <= hi; c++)
+          map.set(c, start.slice(0, -1) + String.fromCharCode(last + (c - lo)));
+      } else {
+        const each = [...m[4].matchAll(/<([0-9a-fA-F]*)>/g)].map((x) => utf16(x[1]));
+        each.forEach((t, k) => lo + k <= hi && map.set(lo + k, t));
+      }
+    }
+  return { map, bytes };
+}
+function fontText(b, f) {
+  if (!f?.map)
+    return bytesText(b);
+  let out = "";
+  for (let i = 0; i < b.length; i += f.bytes) {
+    const code = f.bytes === 2 ? b[i] << 8 | (b[i + 1] ?? 0) : b[i];
+    out += f.map.get(code) ?? (f.bytes === 1 ? String.fromCharCode(code) : "");
+  }
+  return clean(out);
+}
+function clean(s) {
+  let out = "";
+  for (const ch of s) {
+    const c = ch.charCodeAt(0);
+    if (c >= 32 ? c !== 127 : c === 9 || c === 10 || c === 13)
+      out += ch;
+  }
+  return out;
+}
+function contentText(src, fonts = /* @__PURE__ */ new Map(), said = {}) {
   const lines = [];
   let cur = "";
+  let font;
   const newline = () => {
     if (cur.trim())
       lines.push(cur.replace(/\s+/g, " ").trim());
     cur = "";
   };
+  const text = (b) => {
+    if (font?.unreadable) {
+      if (b.length)
+        said.unreadable = true;
+      return "";
+    }
+    return fontText(b, font);
+  };
   const show = (tok) => {
     if (tok?.t === "str")
-      cur += bytesText(tok.v);
+      cur += text(tok.v);
     else if (tok?.t === "arr")
       for (const x of tok.v) {
         if (x.t === "str")
-          cur += bytesText(x.v);
+          cur += text(x.v);
         else if (x.t === "num" && x.v < -200)
           cur += " ";
       }
@@ -2289,7 +2420,7 @@ function contentText(src) {
       i++;
     } else if (c === "/") {
       const m = /^\/[^\s/[\]()<>{}%]*/.exec(src.slice(i, i + 128));
-      push({ t: "other" });
+      push({ t: "name", v: m[0].slice(1) });
       i += m[0].length;
     } else {
       const m = /^[+-]?(\d+\.?\d*|\.\d+)/.exec(src.slice(i, i + 32));
@@ -2323,6 +2454,11 @@ function contentText(src) {
           lastY = y;
           break;
         }
+        case "Tf": {
+          const name = stack[stack.length - 2];
+          font = name?.t === "name" ? fonts.get(name.v) : void 0;
+          break;
+        }
         case "Tj":
         case "TJ":
           show(stack.at(-1));
@@ -2368,7 +2504,13 @@ function pdfDate(s) {
 function readPdf(bytes) {
   const buf = Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength);
   const src = buf.toString("latin1");
-  const objs = objects(src, buf);
+  const decoding = {
+    cache: /* @__PURE__ */ new Map(),
+    texts: /* @__PURE__ */ new Map(),
+    left: MAX_DECODED,
+    textLeft: MAX_TEXT
+  };
+  const objs = objects(src, buf, decoding);
   const trailers = [...src.matchAll(/trailer\s*(<<[\s\S]*?>>)/g)].map((m) => m[1]);
   for (const o of objs.values())
     if (isType(o.dict, "XRef"))
@@ -2428,18 +2570,109 @@ function readPdf(bytes) {
         pages2.push(n);
   }
   const s = new Sections();
+  if (pages2.length > MAX_PAGES) {
+    notes.push(`This PDF has ${pages2.length} pages; the first ${MAX_PAGES} are read.`);
+    pages2.length = MAX_PAGES;
+  }
+  const dictOf = (owner, key) => {
+    const r = ref(owner, key);
+    if (r !== void 0)
+      return objs.get(r)?.dict;
+    const at = new RegExp(`/${key}\\s*<<`).exec(owner);
+    if (!at)
+      return void 0;
+    let depth = 0;
+    for (let k = at.index + at[0].length - 2; k < owner.length - 1; k++) {
+      if (owner[k] === "<" && owner[k + 1] === "<") {
+        depth++;
+        k++;
+      } else if (owner[k] === ">" && owner[k + 1] === ">") {
+        if (--depth === 0)
+          return owner.slice(at.index + at[0].length - 2, k + 2);
+        k++;
+      }
+    }
+    return void 0;
+  };
+  const fontCache = /* @__PURE__ */ new Map();
+  const fontOf = (no) => {
+    const had = fontCache.get(no);
+    if (had)
+      return had;
+    const d = objs.get(no)?.dict ?? "";
+    const type0 = /\/Subtype\s*\/Type0\b/.test(d);
+    const tu = ref(d, "ToUnicode");
+    const cmap = tu !== void 0 ? objs.get(tu) : void 0;
+    const text = cmap ? decodeStream(cmap, decoding)?.toString("latin1") : void 0;
+    const f = text ? (() => {
+      const t = parseToUnicode(text);
+      return { map: t.map, bytes: type0 ? 2 : 1 };
+    })() : type0 ? { bytes: 2, unreadable: true } : { bytes: 1 };
+    fontCache.set(no, f);
+    return f;
+  };
+  const pageFonts = (pageDict) => {
+    const out = /* @__PURE__ */ new Map();
+    let d = pageDict;
+    for (let up = 0; d && up < 32; up++) {
+      const res = dictOf(d, "Resources");
+      const fd = res ? dictOf(res, "Font") : void 0;
+      if (fd) {
+        for (const m of fd.matchAll(/\/([^\s/[\]()<>{}%]+)\s+(\d+)\s+\d+\s+R\b/g))
+          if (!out.has(m[1]))
+            out.set(m[1], fontOf(Number(m[2])));
+        break;
+      }
+      const parent = ref(d, "Parent");
+      d = parent !== void 0 ? objs.get(parent)?.dict : void 0;
+    }
+    return out;
+  };
+  let unreadablePages = 0;
   pages2.forEach((n, i) => {
     const dict = objs.get(n).dict;
+    const fonts = pageFonts(dict);
+    const key = [...fonts].map(([k, f]) => `${k}:${f.map ? "m" : f.unreadable ? "u" : "l"}`).join(",");
+    const said = {};
     const contents = /\/Contents\s*(\[[^\]]*\]|\d+\s+\d+\s+R)/.exec(dict)?.[1] ?? "";
     let streams = refs(contents);
     if (streams.length === 1 && !objs.get(streams[0])?.stream)
       streams = refs(objs.get(streams[0])?.dict ?? "");
-    const text = streams.map((r) => objs.get(r)).map((o) => o ? decodeStream(o)?.toString("latin1") : void 0).filter((t) => t !== void 0).flatMap(contentText);
+    const text = streams.map((r) => objs.get(r)).flatMap((o) => {
+      if (!o)
+        return [];
+      let byFonts = decoding.texts.get(o);
+      if (!byFonts)
+        decoding.texts.set(o, byFonts = /* @__PURE__ */ new Map());
+      let hit = byFonts.get(key);
+      if (!hit) {
+        const t = decodeStream(o, decoding)?.toString("latin1");
+        const one = {};
+        hit = {
+          lines: t === void 0 ? [] : contentText(t, fonts, one),
+          unreadable: !!one.unreadable
+        };
+        byFonts.set(key, hit);
+      }
+      if (hit.unreadable)
+        said.unreadable = true;
+      const lines = hit.lines;
+      decoding.textLeft -= lines.reduce((n2, l) => n2 + l.length, 0);
+      if (decoding.textLeft < 0)
+        throw new ReadError(`This PDF's pages hold more than ${MAX_TEXT / 1024 / 1024} MB of text, so it isn't read`);
+      return lines;
+    });
+    if (said.unreadable)
+      unreadablePages++;
     s.heading("", 0, { page: i + 1 });
     for (const l of text)
       s.add(l);
   });
   const sections = s.done();
+  if (unreadablePages) {
+    notes.push(`${unreadablePages} page${unreadablePages === 1 ? "" : "s"} of this PDF use a font with no map to letters, so some text wasn't read. A person needs to read ${unreadablePages === 1 ? "it" : "them"}.`);
+    meta.flags = ["needs-reading"];
+  }
   if (!sections.length) {
     notes.push(NO_TEXT);
     meta.flags = ["needs-reading"];
@@ -2853,9 +3086,36 @@ function pickIngestor(fileName, head) {
   return best;
 }
 
+// packages/ingestors/dist/traffic-formats.js
+var TRAFFIC_SOURCES = {
+  // A gateway analytics custom report, exported as CSV or JSON: its dimensions and metrics.
+  apigee: {
+    app: ["developer_app", "developer app", "app", "application"],
+    method: ["request_verb", "verb", "method"],
+    path: ["request_path", "request_uri", "uri", "path"],
+    basepath: ["proxy_basepath", "basepath", "base path"],
+    suffix: ["proxy_pathsuffix", "pathsuffix", "path suffix"],
+    proxy: ["apiproxy", "api_proxy", "proxy"],
+    count: ["sum(message_count)", "message_count", "traffic", "count"],
+    time: ["client_received_start_timestamp", "timestamp", "time", "date", "day"]
+  },
+  // An API catalogue's usage export: who calls which operation, and how often.
+  catalog: {
+    app: ["consumer", "consumer app", "application", "app", "client name", "subscriber"],
+    method: ["method", "http method", "verb"],
+    path: ["path", "operation path", "resource", "route"],
+    basepath: ["basepath", "base path"],
+    suffix: [],
+    proxy: ["api", "api name", "proxy", "product"],
+    count: ["calls", "count", "requests", "hits"],
+    time: ["date", "day", "timestamp", "time", "period"]
+  }
+};
+
 export {
   curatedAliases,
   resolveNames,
+  labelText,
   readDrawio,
   drawioIngestor,
   reconcile,
@@ -2898,5 +3158,6 @@ export {
   pdfIngestor,
   figmaIngestor,
   listIngestors,
-  pickIngestor
+  pickIngestor,
+  TRAFFIC_SOURCES
 };