openings 0.1.31 → 0.1.33

package/.codex-plugin/plugin.json

@@ -1,6 +1,6 @@
 {
   "name": "openings",
-  "version": "0.1.31",
+  "version": "0.1.33",
   "description": "Find evidence-grounded jobs, including relevant roles you may not have searched for, without accounts or API keys.",
   "author": { "name": "Openings contributors" },
   "license": "MIT",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "openings",
-  "version": "0.1.31",
+  "version": "0.1.33",
   "description": "A free, candidate-safe job-search substrate for AI agents",
   "license": "MIT",
   "repository": { "type": "git", "url": "git+https://github.com/abhay-avagama/hiring-agent.git" },

package/src/catalog.ts

@@ -1,5 +1,5 @@
 import { CASCADE_MINIMUM, CASCADE_WINDOWS, type Company, type Job, type JobAge, type JobSummary, type SearchQuery, type SearchWindow } from "./types.ts";
-import { providerSpec, type JsonGet } from "./providers.ts";
+import { decodeEntities, providerSpec, type JsonGet } from "./providers.ts";
 import { crawlSite, sitePostingsToJobs } from "./jobposting-site.ts";
 import { classifyJob, isEligibleForCountry, normalizeLocation } from "./locations.ts";
 
@@ -99,12 +99,7 @@ export function createCatalog(options: CatalogOptions): Catalog {
       const company = options.companies.find((candidate) => candidate.ats === ats && candidate.slug === slug);
       if (!company) return null;
       let job = (await fetchJobs(company)).find((candidate) => candidate.id === id) ?? null;
-      if (job && company.ats === "workday" && !job.description.trim()) {
-        const description = await fetchWorkdayDescription(company, job.url, fetcher);
-        job = { ...job, description };
-      }
-      const spec = job && !job.description.trim() ? providerSpec(company.ats) : undefined;
-      if (job && spec?.detail) job = { ...job, description: await spec.detail(company, job, jsonGetter(fetcher, company.name)) };
+      if (job && !job.description.trim()) job = { ...job, description: await fetchJobDescription(company, job, fetcher) };
       if (job && !job.description.trim()) throw new Error(`Full description unavailable for job: ${id}`);
       return job;
     },
@@ -402,6 +397,13 @@ export function workdayPostedAt(label: string | undefined, now: number = Date.no
   return new Date(now - days * 86_400_000).toISOString().slice(0, 10);
 }
 
+/** The full description for a job whose listing carried none (Workday and providers with a detail endpoint); "" when there is no detail source. */
+export async function fetchJobDescription(company: Company, job: Job, fetcher: Fetch): Promise<string> {
+  if (company.ats === "workday") return fetchWorkdayDescription(company, job.url, fetcher);
+  const spec = providerSpec(company.ats);
+  return spec?.detail ? spec.detail(company, job, jsonGetter(fetcher, company.name)) : "";
+}
+
 async function fetchWorkdayDescription(company: Company, jobUrl: string, fetcher: Fetch): Promise<string> {
   const source = parseWorkdayToken(company.token);
   const path = new URL(jobUrl).pathname.replace(new RegExp(`^/en-US/${escapeRegExp(source.site)}`), "");
@@ -516,10 +518,7 @@ function stripHtml(value: string): string {
     .replace(/<br\s*\/?\s*>/gi, "\n")
     .replace(/<\/p>/gi, "\n\n")
     .replace(/<[^>]+>/g, "")
-    .replace(/&nbsp;/g, " ")
-    .replace(/&amp;/g, "&")
-    .replace(/&lt;/g, "<")
-    .replace(/&gt;/g, ">")
+    .replace(/&[#a-z0-9]+;/gi, (entity) => decodeEntities(entity))
     .replace(/\n{3,}/g, "\n\n")
     .trim();
 }

package/src/crawl-reporting.ts

@@ -36,7 +36,8 @@ export function createCrawlReporter(options: { url: string; fetcher?: Fetch; tim
       method: "POST",
       headers: { "content-type": "application/json", "content-encoding": "gzip" },
       body: Bun.gzipSync(JSON.stringify(payload)),
-      signal: AbortSignal.timeout(options.timeoutMs ?? 10_000),
+      // Big employers send tens of megabytes and the aggregator ingests one report at a time; give them room.
+      signal: AbortSignal.timeout(options.timeoutMs ?? 120_000),
     });
     if (!response.ok) throw new Error(`Aggregator rejected crawl report: HTTP ${response.status}`);
   };

package/src/crawler.ts

@@ -1,4 +1,5 @@
 import type { FetchJobsObserver } from "./catalog.ts";
+import { statedExperience } from "./experience.ts";
 import { partitionFor, type Company, type CrawlFailure, type CrawlReport, type CrawlSourceResult, type Job, type JobPartition, type JobSnapshot } from "./types.ts";
 
 export interface SnapshotStore {
@@ -17,6 +18,10 @@ interface CrawlerOptions {
   sourceLimit?: number;
   workdayPageDelayMs?: number;
   workdayCountries?: string[];
+  /** Fetch a job's full description when its listing had none; used to read required experience. */
+  describe?(source: Company, job: Job): Promise<string>;
+  /** Only roles open to these countries, posted in the last 30 days, get a description fetch. Empty: none do. */
+  describeCountries?: string[];
   pacingNow?: () => number;
   pacingSleep?: (delayMs: number) => Promise<void>;
   now?: () => Date;
@@ -40,6 +45,30 @@ export function createCrawler(options: CrawlerOptions): Crawler {
   const pacingSleep = options.pacingSleep ?? ((delayMs: number) => new Promise<void>((resolve) => setTimeout(resolve, delayMs)));
   let previousStart: number | undefined;
   let pacingGate = Promise.resolve();
+  const describeCountries = new Set(options.describeCountries ?? []);
+
+  /** Required experience for every job: from its description, the last crawl's reading, or a paced detail fetch for recent roles in describeCountries. */
+  async function withExperience(source: Company, jobs: Job[], previous: Job[] | undefined): Promise<Job[]> {
+    const known = new Map((previous ?? []).filter((job) => job.experience !== undefined).map((job) => [job.id, job.experience]));
+    const since = now().getTime() - 30 * 86_400_000;
+    const deadline = Date.now() + DESCRIBE_BUDGET_MS;
+    let budget = DESCRIBE_LIMIT;
+    const out: Job[] = [];
+    for (const job of jobs) {
+      if (job.experience !== undefined) out.push(job);
+      else if (job.description.trim()) out.push({ ...job, experience: statedExperience(job.description) });
+      else if (known.has(job.id)) out.push({ ...job, experience: known.get(job.id) });
+      else if (options.describe && budget > 0 && Date.now() < deadline && job.eligibleCountries.some((code) => describeCountries.has(code)) && Date.parse(job.updatedAt ?? "") >= since) {
+        budget -= 1;
+        try {
+          if (options.workdayPageDelayMs) await pacingSleep(options.workdayPageDelayMs);
+          const description = await options.describe(source, job);
+          out.push(description.trim() ? { ...job, experience: statedExperience(description) } : job);
+        } catch { out.push(job); } // unread; the next crawl tries again
+      } else out.push(job);
+    }
+    return out;
+  }
 
   async function paceSourceStart() {
     if (!sourceStartDelayMs) return;
@@ -92,11 +121,12 @@ export function createCrawler(options: CrawlerOptions): Crawler {
             const controller = new AbortController();
             const timer = setTimeout(() => controller.abort(new Error(`Timed out after ${timeoutMs}ms`)), timeoutMs);
             try {
-              const jobs = await options.fetchJobs(source, controller.signal, {
+              const listed = await options.fetchJobs(source, controller.signal, {
                 onBackoff: ({ status, delayMs }) => { metric.backoffMs += delayMs; if (status === 429) metric.throttles += 1; },
                 workdayPageDelayMs: options.workdayPageDelayMs,
                 workdayCountries: options.workdayCountries,
               });
+              const jobs = await withExperience(source, listed, partitionFor(partitions, source.slug)?.jobs);
               partitions[source.slug] = { fetchedAt: now().toISOString(), jobs };
               succeeded += 1;
               metric.status = "succeeded";
@@ -129,13 +159,20 @@ export function createCrawler(options: CrawlerOptions): Crawler {
       await options.store.write({ version: 1, updatedAt: finishedAt, partitions, lastCrawl: report });
       if (options.onCrawled) {
         const crawled = selectedSources.filter((source) => metrics.get(source.slug)!.status === "succeeded");
-        await Promise.all(crawled.map((source) => Promise.resolve().then(() => options.onCrawled!(source, partitions[source.slug]!)).catch(() => undefined)));
+        // Four at a time: sending a thousand reports at once made the largest ones time out while the aggregator queued them.
+        let next = 0;
+        const send = async () => { while (next < crawled.length) { const source = crawled[next++]!; await Promise.resolve().then(() => options.onCrawled!(source, partitions[source.slug]!)).catch(() => undefined); } };
+        await Promise.all(Array.from({ length: Math.min(4, crawled.length) }, send));
       }
       return report;
     },
   };
 }
 
+// ponytail: per-source caps keep one crawl bounded; a big backlog (first run) drains over a few nights.
+const DESCRIBE_LIMIT = 150;
+const DESCRIBE_BUDGET_MS = 180_000;
+
 function partitionTime(value: string | undefined): number {
   const timestamp = Date.parse(value ?? "");
   return Number.isFinite(timestamp) ? timestamp : Number.NEGATIVE_INFINITY;

package/src/experience.ts

@@ -0,0 +1,50 @@
+import { decodeEntities } from "./providers.ts";
+
+/** Years of experience a posting asks for, read from its own text. */
+export interface Experience { min: number; max?: number }
+
+const WORDS: Record<string, number> = { one: 1, two: 2, three: 3, four: 4, five: 5, six: 6, seven: 7, eight: 8, nine: 9, ten: 10, twelve: 12, fifteen: 15 };
+const NUM = String.raw`(\d{1,2}(?:\.\d)?|${Object.keys(WORDS).join("|")})`;
+const RANGE = String.raw`(?<![\d.])${NUM}\s*\+?\s*(?:(?:-|–|—|to)\s*${NUM}\s*)?\+?\s*(?:years?|yrs?)\+?(?:['’]s?)?`;
+// "5+ years of hands-on Java experience" or "Experience: 2-5 years" / "Years of experience 6 to 8 years".
+const YEARS_THEN_EXPERIENCE = new RegExp(String.raw`${RANGE}(?:\s+of)?(?:\s+[\w/&,'’()-]+){0,10}?\s+(?:experiences?|exp)\b`, "gi");
+const EXPERIENCE_THEN_YEARS = new RegExp(String.raw`\b(?:experiences?|exp)\b[^.\n\d]{0,30}?${RANGE}`, "gi");
+// "minimum 15+ years" with no word "experience" after it.
+const MINIMUM_YEARS = new RegExp(String.raw`\b(?:minimum|min\.?|at least|atleast)\s*(?:of\s*)?${RANGE}`, "gi");
+// Company history reads the same way ("our 135 years of experience"); skip matches that talk about the employer.
+const ABOUT_EMPLOYER = /\b(?:our (?:company|firm|group|organi[sz]ation|brands?|history|legacy)|(?:company|firm|group) (?:has|with)|founded|established|history|legacy|heritage)\b[^.\n?!:;•·-]*$/i;
+const MIN_PREFIX = /\b(?:minimum|min\.?|at least|atleast)\s*(?:of\s*)?$/i;
+
+function value(text: string | undefined): number | undefined {
+  if (!text) return undefined;
+  return WORDS[text.toLowerCase()] ?? Number(text);
+}
+
+/** The first experience requirement the text states, or null when it states none. */
+export function statedExperience(description: string): Experience | null {
+  const text = decodeEntities(description);
+  const found: Array<{ index: number; experience: Experience }> = [];
+  for (const pattern of [YEARS_THEN_EXPERIENCE, EXPERIENCE_THEN_YEARS, MINIMUM_YEARS]) {
+    for (const match of text.matchAll(pattern)) {
+      const [low, high] = [value(match[1]), value(match[2])];
+      if (low === undefined || !Number.isFinite(low)) continue;
+      // Requirements rarely pass 20 years; bigger numbers are nearly always an employer's history.
+      if (low > 20 || (high !== undefined && (high < low || high > 40))) continue;
+      const before = text.slice(Math.max(0, match.index - 60), match.index);
+      if (pattern !== MINIMUM_YEARS && ABOUT_EMPLOYER.test(before) && !MIN_PREFIX.test(before)) continue;
+      found.push({ index: match.index, experience: high !== undefined && high !== low ? { min: low, max: high } : { min: low } });
+    }
+  }
+  found.sort((left, right) => left.index - right.index);
+  return found[0]?.experience ?? null;
+}
+
+/** Entry-level titles (intern, trainee, fresher, graduate) as a rough 0–1 year estimate when the posting states nothing. */
+export function titleExperience(title: string): Experience | undefined {
+  return /\b(?:intern|internship|trainee|freshers?|graduate|apprentice(?:ship)?|entry[- ]level)\b/i.test(title) ? { min: 0, max: 1 } : undefined;
+}
+
+/** "2–5 yrs", "5+ yrs". */
+export function experienceLabel(experience: Experience): string {
+  return experience.max === undefined ? `${experience.min}+ yrs` : `${experience.min}–${experience.max} yrs`;
+}

package/src/index.ts

@@ -17,6 +17,7 @@ export const companies: Company[] = Object.entries(companyData).map(([slug, valu
 
 export const catalog = createCatalog({ companies });
 export { createCatalog } from "./catalog.ts";
+export { experienceLabel, statedExperience, titleExperience, type Experience } from "./experience.ts";
 export type { Catalog } from "./catalog.ts";
 export type { Ats, Company, CrawlReport, Job, JobPartition, JobSnapshot, JobSummary, SearchQuery } from "./types.ts";
 export { createCrawlReporter, fetchSeedSnapshot, resolveAggregatorUrl } from "./crawl-reporting.ts";

package/src/job-matching.ts

@@ -1,3 +1,4 @@
+import { experienceLabel } from "./experience.ts";
 import { validateCandidateProfileEvidence, type CandidateProfile } from "./candidate-profile.ts";
 import { isEligibleForCountry, normalizeLocation } from "./locations.ts";
 import { detectRequirementTerms, findTransferability, matchesExactSkillEvidence, requiresExactSkillEvidence, type TransferabilityKind } from "./requirement-vocabulary.ts";
@@ -258,6 +259,13 @@ function supportedRequirements(profile: CandidateProfile, requirements: string[]
 const seniorityTerms = ["manager", "principal", "staff", "lead", "senior", "mid", "junior", "intern"] as const;
 function seniorityAlignment(profile: CandidateProfile, intent: CandidateIntent, job: Job): { score: number; reason?: string } {
   const requested = intent.seniority?.map((value) => value.toLocaleLowerCase()) ?? profile.inferences.filter((inference) => inference.kind === "seniority").map((inference) => inference.value);
+  const resumeYears = profile.inferences.find((inference) => inference.kind === "approximate_experience_years")?.value;
+  if (job.experience && typeof resumeYears === "number" && !intent.seniority?.length) {
+    // The posting's own range is better evidence than a seniority word in the title. A year short or a few over still fits.
+    const asked = experienceLabel(job.experience);
+    const fits = resumeYears >= job.experience.min - 1 && (job.experience.max === undefined || resumeYears <= job.experience.max + 3);
+    return { score: fits ? 3 : -3, reason: `posting asks for ${asked}; resume shows about ${resumeYears} years` };
+  }
   const jobSeniority = seniorityTerms.find((term) => includesPhrase(job.title, term));
   if (!jobSeniority) return { score: 0 };
   if (!requested.length) {

package/src/jobposting-site.ts

@@ -1,5 +1,5 @@
 import { classifyJob } from "./locations.ts";
-import { countryLabel } from "./providers.ts";
+import { countryLabel, decodeEntities } from "./providers.ts";
 import { extractLinks, fetchSafePage, robotsAllows, type PageTransport, type ResolveHost } from "./safe-head.ts";
 import type { Job } from "./types.ts";
 
@@ -158,5 +158,5 @@ export function sitePostingsToJobs(company: { slug: string; name: string }, post
 function hash(value: string): string { let h = 2166136261; for (const ch of value) { h ^= ch.charCodeAt(0); h = Math.imul(h, 16777619) >>> 0; } return h.toString(16); }
 function text(value: unknown): string { return typeof value === "string" ? value.trim() : typeof value === "number" ? String(value) : ""; }
 function isoDate(value: unknown): string | undefined { const time = Date.parse(text(value)); return Number.isFinite(time) ? new Date(time).toISOString() : undefined; }
-function stripHtml(value: string): string { return value.replace(/<[^>]+>/g, " ").replace(/&nbsp;/g, " ").replace(/&amp;/g, "&").replace(/&lt;/g, "<").replace(/&gt;/g, ">").replace(/\s+/g, " ").trim(); }
+function stripHtml(value: string): string { return decodeEntities(value.replace(/<[^>]+>/g, " ")).replace(/\s+/g, " ").trim(); }
 function isRecord(value: unknown): value is Record<string, unknown> { return typeof value === "object" && value !== null && !Array.isArray(value); }

package/src/local-jobs.ts

@@ -15,6 +15,8 @@ interface LocalJobsOptions {
   sourceLimit?: number;
   workdayPageDelayMs?: number;
   workdayCountries?: string[];
+  describe?(source: Company, job: Job): Promise<string>;
+  describeCountries?: string[];
   now?: () => Date;
   onCrawled?(source: Company, partition: JobPartition): void | Promise<void>;
 }

package/src/mcp.ts

@@ -21,6 +21,7 @@ interface ToolHandler {
 export const FLOW_INSTRUCTIONS = [
   "Openings flow: 1) call prepare_job_search for the person's countries; it is normally ready in one call. 2) Ask for the resume before exploring roles, because recommend_jobs ranks by evidence from it. 3) If the person declines a resume, use search_jobs with their keywords. 4) If they decline that too, use your judgement.",
   "Freshness: results start with roles posted in the last 7 days and widen to 14, 30, then everything only when fewer than 5 appear; each result carries age (new, older, stale, undated) and postedDaysAgo. Say which window results came from and present older or stale roles as possibly still open, never as current.",
+  "Experience: a job's experience field is the years its posting states ({ min, max }); null means the posting states none, and absent means it was not read. Quote it as the employer's requirement.",
   "Every match carries applyUrl, the employer's own posting; include it. When recommend_jobs reports no_matches, relay its explanation and nextMoves instead of searching silently.",
 ].join("\n");
 

package/src/providers.ts

@@ -294,10 +294,20 @@ function validToken(value: string | undefined): string | null {
   return value && /^[A-Za-z0-9][A-Za-z0-9._-]*$/.test(value) && !RESERVED_TOKENS.has(value.toLowerCase()) ? value : null;
 }
 
+const NAMED_ENTITIES: Record<string, string> = { nbsp: " ", amp: "&", lt: "<", gt: ">", quot: '"', apos: "'", ndash: "–", mdash: "—", lsquo: "‘", rsquo: "’", ldquo: "“", rdquo: "”", bull: "•", hellip: "…" };
+/** Decode HTML entities in one pass (Workday sends "4&#43; years"), so "&amp;#43;" stays "&#43;" instead of becoming "+". */
+export function decodeEntities(value: string): string {
+  return value.replace(/&(#\d{1,7}|#x[0-9a-f]{1,6}|[a-z]{2,8});/gi, (entity, code: string) => {
+    if (code[0] !== "#") return NAMED_ENTITIES[code.toLowerCase()] ?? entity;
+    const point = code[1] === "x" || code[1] === "X" ? parseInt(code.slice(2), 16) : Number(code.slice(1));
+    return point > 0 && point <= 0x10ffff ? String.fromCodePoint(point) : entity;
+  });
+}
+
 export function plainText(value: string): string {
   return value
     .replace(/<br\s*\/?\s*>/gi, "\n").replace(/<\/(p|li|div|h[1-6])>/gi, "\n").replace(/<[^>]+>/g, "")
-    .replace(/&nbsp;/g, " ").replace(/&amp;/g, "&").replace(/&lt;/g, "<").replace(/&gt;/g, ">").replace(/&quot;/g, '"').replace(/&#39;/g, "'")
+    .replace(/&[#a-z0-9]+;/gi, (entity) => decodeEntities(entity))
     .replace(/[ \t]+\n/g, "\n").replace(/\n{3,}/g, "\n\n").trim();
 }
 

package/src/runtime.ts

@@ -1,7 +1,7 @@
 import { join } from "node:path";
 import type { SnapshotStore } from "./crawler.ts";
 import type { Company, SearchQuery } from "./types.ts";
-import { fetchSourceJobs } from "./catalog.ts";
+import { fetchJobDescription, fetchSourceJobs } from "./catalog.ts";
 import { createCrawlReporter, fetchSeedSnapshot, resolveAggregatorUrl } from "./crawl-reporting.ts";
 import { createUsageReporter, type UsageReporter } from "./usage.ts";
 import { VERSION } from "./version.ts";
@@ -33,6 +33,9 @@ export function createRuntime(options: { dataDir?: string; concurrency?: number;
     sourceLimit: options.sourceLimit,
     workdayPageDelayMs: options.workdayPageDelayMs,
     workdayCountries,
+    // Server crawls set OPENINGS_DESCRIBE_COUNTRIES (IN) to read required experience from Workday detail pages; installs skip the extra requests.
+    describe: (source, job) => fetchJobDescription(source, job, globalThis.fetch),
+    describeCountries: (process.env.OPENINGS_DESCRIBE_COUNTRIES ?? "").split(",").map((code) => code.trim().toUpperCase()).filter((code) => /^[A-Z]{2}$/.test(code)),
     onCrawled,
   });
   const recommender = createJobRecommender({ sources: companies, store, crawl: local.crawl });

package/src/types.ts

@@ -66,6 +66,8 @@ export interface SourceVerificationResult {
   rejected: RejectedSource[];
 }
 
+import type { Experience } from "./experience.ts";
+
 export type WorkMode = "remote" | "hybrid" | "onsite" | "unknown";
 export type EligibilityConfidence = "explicit" | "inferred" | "unknown";
 
@@ -83,6 +85,8 @@ export interface JobSummary {
   url: string;
   /** Posting date when the board exposes one (Workday's relative label is approximate), otherwise the board's last-update time. */
   updatedAt?: string;
+  /** Years of experience the posting states, read from its description. null: the description states none. Absent: not read yet. */
+  experience?: Experience | null;
   /** Days since the posting date at the time of the search; absent when the board gave no date. */
   postedDaysAgo?: number;
   /** new = 7 days or less, older = 8 to 30, stale = beyond 30, undated = no posting date. Present it as such; a stale listing may still be open. */

package/src/version.ts

@@ -1 +1 @@
-export const VERSION = "0.1.31";
+export const VERSION = "0.1.33";