fullstackgtm 0.39.0 → 0.41.0

package/src/connectors/atsBoards.ts

@@ -0,0 +1,242 @@
+/**
+ * ATS job-board adapters for the `signals` layer — the free, no-auth example of
+ * "watch an account for hiring motion." Each public board JSON endpoint turns a
+ * company's board token into a list of open roles; the signals layer maps each
+ * role to a `job` signal whose `quote` is the role title plus a JD snippet that
+ * VERBATIM contains a configured keyword (so the downstream judge/drafter can
+ * gate a why-now on something a human can verify).
+ *
+ * Mirrors the `prospectSources.ts` raw-fetch idiom exactly: zero runtime deps
+ * (global fetch only, injectable for tests), a base-url override per provider,
+ * and a per-request `try { } catch { continue }` so one provider's outage never
+ * sinks a multi-source fetch. No SDK, no key.
+ *
+ * Governance: read-only. These adapters fetch public board JSON and return open
+ * roles. They write nothing — the signals layer (and only with `--save`) is the
+ * one that persists, and it never touches the CRM.
+ */
+
+export type AtsBoardSource = "greenhouse" | "lever" | "ashby";
+
+/** One open role, normalized across the three providers. */
+export type AtsJob = {
+  /** Role title, verbatim from the board. */
+  title: string;
+  /**
+   * A short JD snippet. When `keywords` are supplied, this is a window around
+   * the first matching keyword (kept verbatim so the keyword survives into the
+   * evidence quote); otherwise it is the leading slice of the description.
+   */
+  jdSnippet: string;
+  /** Canonical posting URL (the signal's sourceUrl). */
+  url: string;
+  /**
+   * Stable per-role identity within a board (provider id when present, else the
+   * title). Used by the reposted-role heuristic to tell "same role, gone, back"
+   * from "two roles that happen to share a title".
+   */
+  firstSeenKey: string;
+};
+
+type FetchImpl = typeof fetch;
+
+const DEFAULT_BASE_URLS: Record<AtsBoardSource, string> = {
+  greenhouse: "https://boards-api.greenhouse.io",
+  lever: "https://api.lever.co",
+  ashby: "https://api.ashbyhq.com",
+};
+
+/** JD snippet length — long enough to carry a verbatim keyword window. */
+const SNIPPET_LEN = 240;
+
+/**
+ * Fetch open roles for one board. Network failures (offline, non-2xx, malformed
+ * JSON) yield `[]` rather than throwing, matching the per-source resilience of
+ * the prospect sources — a single board's outage must not abort a watchlist run.
+ */
+export async function fetchAtsJobs(opts: {
+  source: AtsBoardSource;
+  boardToken: string;
+  accountDomain: string;
+  keywords?: string[];
+  fetchImpl?: FetchImpl;
+  apiBaseUrl?: string;
+}): Promise<AtsJob[]> {
+  const fetchImpl = opts.fetchImpl ?? fetch;
+  const base = (opts.apiBaseUrl ?? DEFAULT_BASE_URLS[opts.source]).replace(/\/$/, "");
+  const token = opts.boardToken.trim();
+  if (!token) return [];
+  const keywords = opts.keywords ?? [];
+  try {
+    if (opts.source === "greenhouse") {
+      return await fetchGreenhouse(fetchImpl, base, token, keywords);
+    }
+    if (opts.source === "lever") {
+      return await fetchLever(fetchImpl, base, token, keywords);
+    }
+    return await fetchAshby(fetchImpl, base, token, keywords);
+  } catch {
+    // Offline / 5xx / malformed payload: an empty board, never a crash.
+    return [];
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Greenhouse — GET /v1/boards/<token>/jobs?content=true
+
+async function fetchGreenhouse(
+  fetchImpl: FetchImpl,
+  base: string,
+  token: string,
+  keywords: string[],
+): Promise<AtsJob[]> {
+  const response = await fetchImpl(`${base}/v1/boards/${encodeURIComponent(token)}/jobs?content=true`, {
+    headers: { Accept: "application/json" },
+  });
+  if (!response.ok) return [];
+  const data = (await response.json()) as {
+    jobs?: Array<{ id?: number | string; title?: string; absolute_url?: string; content?: string }>;
+  };
+  const out: AtsJob[] = [];
+  for (const job of data.jobs ?? []) {
+    const title = strOrEmpty(job.title);
+    if (!title) continue;
+    const description = decodeHtmlish(strOrEmpty(job.content));
+    out.push({
+      title,
+      jdSnippet: snippetFor(description, keywords),
+      url: strOrEmpty(job.absolute_url),
+      firstSeenKey: job.id != null ? String(job.id) : title,
+    });
+  }
+  return out;
+}
+
+// ---------------------------------------------------------------------------
+// Lever — GET /v0/postings/<token>?mode=json (returns a bare array)
+
+async function fetchLever(
+  fetchImpl: FetchImpl,
+  base: string,
+  token: string,
+  keywords: string[],
+): Promise<AtsJob[]> {
+  const response = await fetchImpl(`${base}/v0/postings/${encodeURIComponent(token)}?mode=json`, {
+    headers: { Accept: "application/json" },
+  });
+  if (!response.ok) return [];
+  const data = (await response.json()) as Array<{
+    id?: string;
+    text?: string;
+    hostedUrl?: string;
+    descriptionPlain?: string;
+    description?: string;
+  }>;
+  const out: AtsJob[] = [];
+  for (const posting of Array.isArray(data) ? data : []) {
+    const title = strOrEmpty(posting.text);
+    if (!title) continue;
+    const description = strOrEmpty(posting.descriptionPlain) || decodeHtmlish(strOrEmpty(posting.description));
+    out.push({
+      title,
+      jdSnippet: snippetFor(description, keywords),
+      url: strOrEmpty(posting.hostedUrl),
+      firstSeenKey: strOrEmpty(posting.id) || title,
+    });
+  }
+  return out;
+}
+
+// ---------------------------------------------------------------------------
+// Ashby — POST /posting-api/job-board/<token> with { jobBoardName }
+
+async function fetchAshby(
+  fetchImpl: FetchImpl,
+  base: string,
+  token: string,
+  keywords: string[],
+): Promise<AtsJob[]> {
+  const response = await fetchImpl(`${base}/posting-api/job-board/${encodeURIComponent(token)}`, {
+    method: "POST",
+    headers: { "Content-Type": "application/json", Accept: "application/json" },
+    body: JSON.stringify({ jobBoardName: token, includeCompensation: false }),
+  });
+  if (!response.ok) return [];
+  const data = (await response.json()) as {
+    jobs?: Array<{
+      id?: string;
+      title?: string;
+      jobUrl?: string;
+      applyUrl?: string;
+      descriptionPlain?: string;
+      descriptionHtml?: string;
+    }>;
+  };
+  const out: AtsJob[] = [];
+  for (const job of data.jobs ?? []) {
+    const title = strOrEmpty(job.title);
+    if (!title) continue;
+    const description = strOrEmpty(job.descriptionPlain) || decodeHtmlish(strOrEmpty(job.descriptionHtml));
+    out.push({
+      title,
+      jdSnippet: snippetFor(description, keywords),
+      url: strOrEmpty(job.jobUrl) || strOrEmpty(job.applyUrl),
+      firstSeenKey: strOrEmpty(job.id) || title,
+    });
+  }
+  return out;
+}
+
+// ---------------------------------------------------------------------------
+// Snippet selection: a verbatim window around the first matching keyword.
+
+/**
+ * Pick a verbatim JD snippet. If a keyword matches the description (case-
+ * insensitive), return a window centered on the first match — kept verbatim so
+ * the keyword text survives into the signal quote and the evidence gate holds.
+ * With no keywords (or no match), fall back to the leading slice. Returns "" for
+ * an empty description so the caller can decide whether the role still qualifies.
+ */
+export function snippetFor(description: string, keywords: string[]): string {
+  const text = description.trim();
+  if (!text) return "";
+  for (const keyword of keywords) {
+    const needle = keyword.trim();
+    if (!needle) continue;
+    const idx = text.toLowerCase().indexOf(needle.toLowerCase());
+    if (idx === -1) continue;
+    const pad = Math.max(0, Math.floor((SNIPPET_LEN - needle.length) / 2));
+    let start = Math.max(0, idx - pad);
+    let end = Math.min(text.length, idx + needle.length + pad);
+    // Snap to word boundaries so the window doesn't start/end mid-word, but
+    // never past the keyword itself (the verbatim span must stay intact).
+    while (start > 0 && /\S/.test(text[start - 1]) && start < idx) start += 1;
+    while (end < text.length && /\S/.test(text[end]) && end > idx + needle.length) end -= 1;
+    return text.slice(start, end).trim();
+  }
+  return text.slice(0, SNIPPET_LEN).trim();
+}
+
+/**
+ * Minimal HTML-ish cleanup for board `content`/`description` HTML: drop tags and
+ * decode the handful of entities Greenhouse/Ashby emit. Dependency-free; good
+ * enough to surface a readable, keyword-matchable snippet (the quote does not
+ * need to be byte-perfect HTML, only verbatim text the keyword appears in).
+ */
+function decodeHtmlish(raw: string): string {
+  return raw
+    .replace(/<\s*br\s*\/?\s*>/gi, " ")
+    .replace(/<\/?[^>]+>/g, " ")
+    .replace(/&amp;/g, "&")
+    .replace(/&lt;/g, "<")
+    .replace(/&gt;/g, ">")
+    .replace(/&quot;/g, '"')
+    .replace(/&#39;|&apos;/g, "'")
+    .replace(/&nbsp;/g, " ")
+    .replace(/\s+/g, " ")
+    .trim();
+}
+
+function strOrEmpty(value: unknown): string {
+  return typeof value === "string" ? value.trim() : "";
+}

package/src/connectors/prospectSources.ts

@@ -180,6 +180,52 @@ function normalizeLinkedin(value: string | undefined): string | undefined {
  * full name + a company domain (or name). Returns the prospects with `email`
  * filled where the waterfall found one; rows with no hit are returned unchanged.
  */
+const sleep = (ms: number) => new Promise<void>((resolve) => setTimeout(resolve, ms));
+
+/** Run `count` async tasks with at most `limit` in flight. Order-independent. */
+async function runWithConcurrency<T>(count: number, limit: number, worker: (index: number) => Promise<T>): Promise<T[]> {
+  const out: T[] = new Array(count);
+  let next = 0;
+  const runners = Array.from({ length: Math.max(1, Math.min(limit, count || 1)) }, async () => {
+    for (let i = next++; i < count; i = next++) out[i] = await worker(i);
+  });
+  await Promise.all(runners);
+  return out;
+}
+
+/**
+ * POST a pipe0 sync run with exponential backoff on transient failure
+ * (throttle/5xx/network). pipe0's waterfall throttles under burst — a 429'd
+ * chunk returns all-failed — so concurrency must be paired with real backoff,
+ * not a single retry, or coverage collapses at scale (a parallel run without
+ * this dropped the work-email hit-rate from ~79% to ~17%). Retries 500ms →
+ * 1s → 2s → 4s (capped) so a throttled chunk lands on a later attempt instead
+ * of being lost.
+ */
+async function pipe0Post(
+  fetchImpl: FetchImpl,
+  base: string,
+  apiKey: string,
+  payload: unknown,
+  maxAttempts = 5,
+): Promise<Pipe0RunResponse | undefined> {
+  for (let attempt = 0; attempt < maxAttempts; attempt++) {
+    try {
+      const response = await fetchImpl(`${base}/v1/pipes/run/sync`, {
+        method: "POST",
+        headers: { Authorization: `Bearer ${apiKey}`, "Content-Type": "application/json" },
+        body: JSON.stringify(payload),
+      });
+      if (response.ok) return (await response.json()) as Pipe0RunResponse;
+      // non-ok (429/5xx) — fall through to backoff + retry
+    } catch {
+      // network error — fall through to backoff + retry
+    }
+    if (attempt < maxAttempts - 1) await sleep(Math.min(8000, 500 * 2 ** attempt));
+  }
+  return undefined; // still failing after backoff — caller keeps the other chunks
+}
+
 export async function pipe0ResolveWorkEmails(opts: {
   apiKey: string;
   prospects: Prospect[];
@@ -189,43 +235,48 @@ export async function pipe0ResolveWorkEmails(opts: {
    *  batch rate-limit (a throttled chunk returns work_email status "failed"
    *  for every row, so one big batch is all-or-nothing). Default 3. */
   chunkSize?: number;
+  /** Chunks resolved in parallel, bounded to respect pipe0's rate limit. Default
+   *  3 — paired with pipe0Post's backoff this cuts wall-clock from O(n) serial
+   *  calls without throttling away coverage (higher defaults tanked the hit-rate
+   *  at scale). */
+  concurrency?: number;
 }): Promise<Prospect[]> {
   const fetchImpl = opts.fetchImpl ?? fetch;
   const base = (opts.apiBaseUrl ?? "https://api.pipe0.com").replace(/\/$/, "");
   const chunkSize = Math.max(1, opts.chunkSize ?? 3);
+  const concurrency = Math.max(1, opts.concurrency ?? 3);
 
   // Only rows with the waterfall's required inputs (name + company_domain|name).
   const resolvable = opts.prospects.filter((p) => p.fullName && (p.companyDomain || p.companyName));
   if (resolvable.length === 0) return opts.prospects;
 
-  const emailByKey = new Map<string, string>();
-  for (let i = 0; i < resolvable.length; i += chunkSize) {
-    const chunk = resolvable.slice(i, i + chunkSize);
-    const input = chunk.map((p) => ({
+  const chunks: Prospect[][] = [];
+  for (let i = 0; i < resolvable.length; i += chunkSize) chunks.push(resolvable.slice(i, i + chunkSize));
+
+  const perChunk = await runWithConcurrency(chunks.length, concurrency, async (ci) => {
+    const input = chunks[ci].map((p) => ({
       name: p.fullName,
       ...(p.companyDomain ? { company_domain: p.companyDomain } : { company_name: p.companyName }),
     }));
-    let body: Pipe0RunResponse;
-    try {
-      const response = await fetchImpl(`${base}/v1/pipes/run/sync`, {
-        method: "POST",
-        headers: { Authorization: `Bearer ${opts.apiKey}`, "Content-Type": "application/json" },
-        body: JSON.stringify({ pipes: [{ pipe_id: "person:workemail:waterfall@1" }], input }),
-      });
-      if (!response.ok) continue; // throttled/5xx chunk — skip, keep the rest
-      body = (await response.json()) as Pipe0RunResponse;
-    } catch {
-      continue; // network hiccup on one chunk must not sink the whole run
-    }
-    for (const recordId of body.order ?? []) {
-      const fields = body.records?.[recordId]?.fields ?? {};
+    const body = await pipe0Post(fetchImpl, base, opts.apiKey, {
+      pipes: [{ pipe_id: "person:workemail:waterfall@1" }],
+      input,
+    });
+    const entries: Array<[string, string]> = [];
+    for (const recordId of body?.order ?? []) {
+      const fields = body!.records?.[recordId]?.fields ?? {};
       const email = fields.work_email?.status === "completed" ? fieldValue(fields.work_email) : undefined;
       if (email) {
         const domain = fieldValue(fields.company_domain) ?? fieldValue(fields.company_name);
-        emailByKey.set(personKey(fieldValue(fields.name), domain), email);
+        entries.push([personKey(fieldValue(fields.name), domain), email]);
       }
     }
-  }
+    return entries;
+  });
+
+  const emailByKey = new Map<string, string>();
+  for (const entries of perChunk) for (const [k, v] of entries) emailByKey.set(k, v);
+
   return opts.prospects.map((p) => {
     const email = emailByKey.get(personKey(p.fullName, p.companyDomain ?? p.companyName));
     return email ? { ...p, email } : p;
@@ -262,10 +313,13 @@ export async function pipe0ResolveCompanyDomains(opts: {
   apiBaseUrl?: string;
   fetchImpl?: FetchImpl;
   chunkSize?: number;
+  /** Chunks resolved in parallel (bounded). Default 3. */
+  concurrency?: number;
 }): Promise<Prospect[]> {
   const fetchImpl = opts.fetchImpl ?? fetch;
   const base = (opts.apiBaseUrl ?? "https://api.pipe0.com").replace(/\/$/, "");
   const chunkSize = Math.max(1, opts.chunkSize ?? 5);
+  const concurrency = Math.max(1, opts.concurrency ?? 3);
 
   const uniqueNames = [
     ...new Set(
@@ -277,30 +331,27 @@ export async function pipe0ResolveCompanyDomains(opts: {
   ];
   if (uniqueNames.length === 0) return opts.prospects;
 
-  const domainByName = new Map<string, string>();
-  for (let i = 0; i < uniqueNames.length; i += chunkSize) {
-    const chunk = uniqueNames.slice(i, i + chunkSize);
-    try {
-      const response = await fetchImpl(`${base}/v1/pipes/run/sync`, {
-        method: "POST",
-        headers: { Authorization: `Bearer ${opts.apiKey}`, "Content-Type": "application/json" },
-        body: JSON.stringify({
-          pipes: [{ pipe_id: "company:identity@1" }],
-          input: chunk.map((company_name) => ({ company_name })),
-        }),
-      });
-      if (!response.ok) continue;
-      const body = (await response.json()) as Pipe0RunResponse;
-      for (const recordId of body.order ?? []) {
-        const fields = body.records?.[recordId]?.fields ?? {};
-        const name = fieldValue(fields.company_name) ?? fieldValue(fields.cleaned_company_name);
-        const domain = hostFromUrl(fieldValue(fields.company_website_url));
-        if (name && domain) domainByName.set(name.trim().toLowerCase(), domain);
-      }
-    } catch {
-      continue;
+  const chunks: string[][] = [];
+  for (let i = 0; i < uniqueNames.length; i += chunkSize) chunks.push(uniqueNames.slice(i, i + chunkSize));
+
+  const perChunk = await runWithConcurrency(chunks.length, concurrency, async (ci) => {
+    const body = await pipe0Post(fetchImpl, base, opts.apiKey, {
+      pipes: [{ pipe_id: "company:identity@1" }],
+      input: chunks[ci].map((company_name) => ({ company_name })),
+    });
+    const entries: Array<[string, string]> = [];
+    for (const recordId of body?.order ?? []) {
+      const fields = body!.records?.[recordId]?.fields ?? {};
+      const name = fieldValue(fields.company_name) ?? fieldValue(fields.cleaned_company_name);
+      const domain = hostFromUrl(fieldValue(fields.company_website_url));
+      if (name && domain) entries.push([name.trim().toLowerCase(), domain]);
     }
-  }
+    return entries;
+  });
+
+  const domainByName = new Map<string, string>();
+  for (const entries of perChunk) for (const [k, v] of entries) domainByName.set(k, v);
+
   return opts.prospects.map((p) => {
     if (p.companyDomain || !p.companyName) return p;
     const domain = domainByName.get(p.companyName.trim().toLowerCase());