openings 0.1.32 → 0.1.33

package/.codex-plugin/plugin.json

@@ -1,6 +1,6 @@
 {
   "name": "openings",
-  "version": "0.1.32",
+  "version": "0.1.33",
   "description": "Find evidence-grounded jobs, including relevant roles you may not have searched for, without accounts or API keys.",
   "author": { "name": "Openings contributors" },
   "license": "MIT",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "openings",
-  "version": "0.1.32",
+  "version": "0.1.33",
   "description": "A free, candidate-safe job-search substrate for AI agents",
   "license": "MIT",
   "repository": { "type": "git", "url": "git+https://github.com/abhay-avagama/hiring-agent.git" },

package/src/catalog.ts

@@ -1,5 +1,5 @@
 import { CASCADE_MINIMUM, CASCADE_WINDOWS, type Company, type Job, type JobAge, type JobSummary, type SearchQuery, type SearchWindow } from "./types.ts";
-import { providerSpec, type JsonGet } from "./providers.ts";
+import { decodeEntities, providerSpec, type JsonGet } from "./providers.ts";
 import { crawlSite, sitePostingsToJobs } from "./jobposting-site.ts";
 import { classifyJob, isEligibleForCountry, normalizeLocation } from "./locations.ts";
 
@@ -518,10 +518,7 @@ function stripHtml(value: string): string {
     .replace(/<br\s*\/?\s*>/gi, "\n")
     .replace(/<\/p>/gi, "\n\n")
     .replace(/<[^>]+>/g, "")
-    .replace(/&nbsp;/g, " ")
-    .replace(/&amp;/g, "&")
-    .replace(/&lt;/g, "<")
-    .replace(/&gt;/g, ">")
+    .replace(/&[#a-z0-9]+;/gi, (entity) => decodeEntities(entity))
     .replace(/\n{3,}/g, "\n\n")
     .trim();
 }

package/src/crawl-reporting.ts

@@ -36,7 +36,8 @@ export function createCrawlReporter(options: { url: string; fetcher?: Fetch; tim
       method: "POST",
       headers: { "content-type": "application/json", "content-encoding": "gzip" },
       body: Bun.gzipSync(JSON.stringify(payload)),
-      signal: AbortSignal.timeout(options.timeoutMs ?? 10_000),
+      // Big employers send tens of megabytes and the aggregator ingests one report at a time; give them room.
+      signal: AbortSignal.timeout(options.timeoutMs ?? 120_000),
     });
     if (!response.ok) throw new Error(`Aggregator rejected crawl report: HTTP ${response.status}`);
   };

package/src/crawler.ts

@@ -159,7 +159,10 @@ export function createCrawler(options: CrawlerOptions): Crawler {
       await options.store.write({ version: 1, updatedAt: finishedAt, partitions, lastCrawl: report });
       if (options.onCrawled) {
         const crawled = selectedSources.filter((source) => metrics.get(source.slug)!.status === "succeeded");
-        await Promise.all(crawled.map((source) => Promise.resolve().then(() => options.onCrawled!(source, partitions[source.slug]!)).catch(() => undefined)));
+        // Four at a time: sending a thousand reports at once made the largest ones time out while the aggregator queued them.
+        let next = 0;
+        const send = async () => { while (next < crawled.length) { const source = crawled[next++]!; await Promise.resolve().then(() => options.onCrawled!(source, partitions[source.slug]!)).catch(() => undefined); } };
+        await Promise.all(Array.from({ length: Math.min(4, crawled.length) }, send));
       }
       return report;
     },

package/src/experience.ts

@@ -1,14 +1,18 @@
+import { decodeEntities } from "./providers.ts";
+
 /** Years of experience a posting asks for, read from its own text. */
 export interface Experience { min: number; max?: number }
 
 const WORDS: Record<string, number> = { one: 1, two: 2, three: 3, four: 4, five: 5, six: 6, seven: 7, eight: 8, nine: 9, ten: 10, twelve: 12, fifteen: 15 };
 const NUM = String.raw`(\d{1,2}(?:\.\d)?|${Object.keys(WORDS).join("|")})`;
-const RANGE = String.raw`(?<![\d.])${NUM}\s*\+?\s*(?:(?:-|–|—|to)\s*${NUM}\s*)?\+?\s*(?:years?|yrs?)(?:'|’)?`;
+const RANGE = String.raw`(?<![\d.])${NUM}\s*\+?\s*(?:(?:-|–|—|to)\s*${NUM}\s*)?\+?\s*(?:years?|yrs?)\+?(?:['’]s?)?`;
 // "5+ years of hands-on Java experience" or "Experience: 2-5 years" / "Years of experience 6 to 8 years".
-const YEARS_THEN_EXPERIENCE = new RegExp(String.raw`${RANGE}(?:\s+of)?(?:\s+[\w/&,'’()-]+){0,6}?\s+(?:experience|exp)\b`, "gi");
-const EXPERIENCE_THEN_YEARS = new RegExp(String.raw`\b(?:experience|exp)\b[^.\n\d]{0,30}?${RANGE}`, "gi");
+const YEARS_THEN_EXPERIENCE = new RegExp(String.raw`${RANGE}(?:\s+of)?(?:\s+[\w/&,'’()-]+){0,10}?\s+(?:experiences?|exp)\b`, "gi");
+const EXPERIENCE_THEN_YEARS = new RegExp(String.raw`\b(?:experiences?|exp)\b[^.\n\d]{0,30}?${RANGE}`, "gi");
+// "minimum 15+ years" with no word "experience" after it.
+const MINIMUM_YEARS = new RegExp(String.raw`\b(?:minimum|min\.?|at least|atleast)\s*(?:of\s*)?${RANGE}`, "gi");
 // Company history reads the same way ("our 135 years of experience"); skip matches that talk about the employer.
-const ABOUT_EMPLOYER = /\b(?:we|our|company|firm|founded|established|history|legacy|heritage)\b[^.\n?!:;•·-]*$/i;
+const ABOUT_EMPLOYER = /\b(?:our (?:company|firm|group|organi[sz]ation|brands?|history|legacy)|(?:company|firm|group) (?:has|with)|founded|established|history|legacy|heritage)\b[^.\n?!:;•·-]*$/i;
 const MIN_PREFIX = /\b(?:minimum|min\.?|at least|atleast)\s*(?:of\s*)?$/i;
 
 function value(text: string | undefined): number | undefined {
@@ -17,16 +21,17 @@ function value(text: string | undefined): number | undefined {
 }
 
 /** The first experience requirement the text states, or null when it states none. */
-export function statedExperience(text: string): Experience | null {
+export function statedExperience(description: string): Experience | null {
+  const text = decodeEntities(description);
   const found: Array<{ index: number; experience: Experience }> = [];
-  for (const pattern of [YEARS_THEN_EXPERIENCE, EXPERIENCE_THEN_YEARS]) {
+  for (const pattern of [YEARS_THEN_EXPERIENCE, EXPERIENCE_THEN_YEARS, MINIMUM_YEARS]) {
     for (const match of text.matchAll(pattern)) {
       const [low, high] = [value(match[1]), value(match[2])];
       if (low === undefined || !Number.isFinite(low)) continue;
       // Requirements rarely pass 20 years; bigger numbers are nearly always an employer's history.
       if (low > 20 || (high !== undefined && (high < low || high > 40))) continue;
       const before = text.slice(Math.max(0, match.index - 60), match.index);
-      if (ABOUT_EMPLOYER.test(before) && !MIN_PREFIX.test(before)) continue;
+      if (pattern !== MINIMUM_YEARS && ABOUT_EMPLOYER.test(before) && !MIN_PREFIX.test(before)) continue;
       found.push({ index: match.index, experience: high !== undefined && high !== low ? { min: low, max: high } : { min: low } });
     }
   }

package/src/jobposting-site.ts

@@ -1,5 +1,5 @@
 import { classifyJob } from "./locations.ts";
-import { countryLabel } from "./providers.ts";
+import { countryLabel, decodeEntities } from "./providers.ts";
 import { extractLinks, fetchSafePage, robotsAllows, type PageTransport, type ResolveHost } from "./safe-head.ts";
 import type { Job } from "./types.ts";
 
@@ -158,5 +158,5 @@ export function sitePostingsToJobs(company: { slug: string; name: string }, post
 function hash(value: string): string { let h = 2166136261; for (const ch of value) { h ^= ch.charCodeAt(0); h = Math.imul(h, 16777619) >>> 0; } return h.toString(16); }
 function text(value: unknown): string { return typeof value === "string" ? value.trim() : typeof value === "number" ? String(value) : ""; }
 function isoDate(value: unknown): string | undefined { const time = Date.parse(text(value)); return Number.isFinite(time) ? new Date(time).toISOString() : undefined; }
-function stripHtml(value: string): string { return value.replace(/<[^>]+>/g, " ").replace(/&nbsp;/g, " ").replace(/&amp;/g, "&").replace(/&lt;/g, "<").replace(/&gt;/g, ">").replace(/\s+/g, " ").trim(); }
+function stripHtml(value: string): string { return decodeEntities(value.replace(/<[^>]+>/g, " ")).replace(/\s+/g, " ").trim(); }
 function isRecord(value: unknown): value is Record<string, unknown> { return typeof value === "object" && value !== null && !Array.isArray(value); }

package/src/providers.ts

@@ -294,10 +294,20 @@ function validToken(value: string | undefined): string | null {
   return value && /^[A-Za-z0-9][A-Za-z0-9._-]*$/.test(value) && !RESERVED_TOKENS.has(value.toLowerCase()) ? value : null;
 }
 
+const NAMED_ENTITIES: Record<string, string> = { nbsp: " ", amp: "&", lt: "<", gt: ">", quot: '"', apos: "'", ndash: "–", mdash: "—", lsquo: "‘", rsquo: "’", ldquo: "“", rdquo: "”", bull: "•", hellip: "…" };
+/** Decode HTML entities in one pass (Workday sends "4&#43; years"), so "&amp;#43;" stays "&#43;" instead of becoming "+". */
+export function decodeEntities(value: string): string {
+  return value.replace(/&(#\d{1,7}|#x[0-9a-f]{1,6}|[a-z]{2,8});/gi, (entity, code: string) => {
+    if (code[0] !== "#") return NAMED_ENTITIES[code.toLowerCase()] ?? entity;
+    const point = code[1] === "x" || code[1] === "X" ? parseInt(code.slice(2), 16) : Number(code.slice(1));
+    return point > 0 && point <= 0x10ffff ? String.fromCodePoint(point) : entity;
+  });
+}
+
 export function plainText(value: string): string {
   return value
     .replace(/<br\s*\/?\s*>/gi, "\n").replace(/<\/(p|li|div|h[1-6])>/gi, "\n").replace(/<[^>]+>/g, "")
-    .replace(/&nbsp;/g, " ").replace(/&amp;/g, "&").replace(/&lt;/g, "<").replace(/&gt;/g, ">").replace(/&quot;/g, '"').replace(/&#39;/g, "'")
+    .replace(/&[#a-z0-9]+;/gi, (entity) => decodeEntities(entity))
     .replace(/[ \t]+\n/g, "\n").replace(/\n{3,}/g, "\n\n").trim();
 }
 

package/src/version.ts

@@ -1 +1 @@
-export const VERSION = "0.1.32";
+export const VERSION = "0.1.33";