openings 0.1.45 → 0.1.47

package/.codex-plugin/plugin.json

@@ -1,6 +1,6 @@
 {
   "name": "openings",
-  "version": "0.1.45",
+  "version": "0.1.46",
   "description": "Find evidence-grounded jobs, including relevant roles you may not have searched for, without accounts or API keys.",
   "author": { "name": "Openings contributors" },
   "license": "MIT",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "openings",
-  "version": "0.1.45",
+  "version": "0.1.47",
   "description": "A free, candidate-safe job-search substrate for AI agents",
   "license": "MIT",
   "repository": { "type": "git", "url": "git+https://github.com/abhay-avagama/hiring-agent.git" },

package/src/catalog.ts

@@ -1,6 +1,7 @@
 import { CASCADE_MINIMUM, CASCADE_WINDOWS, type Company, type Job, type JobAge, type JobSummary, type SearchQuery, type SearchWindow } from "./types.ts";
 import { decodeEntities, providerSpec, type JsonGet } from "./providers.ts";
 import { matchesSearchTerms, searchHaystack, searchTerms } from "./text-match.ts";
+import { extractSkills } from "./skills.ts";
 import { crawlSite, sitePostingsToJobs } from "./jobposting-site.ts";
 import { classifyJob, isEligibleForCountry, normalizeLocation } from "./locations.ts";
 import { experienceMatches, normalizeJobExperience } from "./experience.ts";
@@ -518,7 +519,9 @@ function matches(job: Job, query: SearchQuery): boolean {
   const terms = searchTerms(query.query);
   const location = query.location ? normalizeLocation(query.location) : undefined;
   // Whole tokens only: a substring match once made "ios" find "Axio Biosolutions".
-  const searchable = searchHaystack(`${job.title} ${job.company}`);
+  // A skill stated only in the description is what most searches are for, so the description's vocabulary
+  // terms join the title and employer. The hosted index sends them precomputed; a local crawl reads them here.
+  const searchable = searchHaystack(`${job.title} ${job.company} ${job.skills ?? extractSkills(job.description)}`);
   return (terms.length === 0 || matchesSearchTerms(searchable, terms))
     && (!location || normalizeLocation(job.location).includes(location))
     && (!query.country || isEligibleForCountry(job, query.country))

package/src/crawl-reporting.ts

@@ -27,19 +27,37 @@ export function resolveAggregatorUrl(value: string | undefined): string | undefi
 }
 
 /** Posts each successfully crawled partition to the aggregator. Only job data is sent, never resume content. */
-export function createCrawlReporter(options: { url: string; fetcher?: Fetch; timeoutMs?: number }) {
+export function createCrawlReporter(options: { url: string; fetcher?: Fetch; timeoutMs?: number; attempts?: number; sleep?: (ms: number) => Promise<void> }) {
   const fetcher = options.fetcher ?? globalThis.fetch;
   const target = endpoint(options.url, "v1/crawls");
+  const attempts = Math.max(1, options.attempts ?? 3);
+  const sleep = options.sleep ?? ((ms: number) => new Promise<void>((resolve) => setTimeout(resolve, ms)));
   return async (source: Company, partition: JobPartition): Promise<void> => {
     const payload: CrawlReportPayload = { version: 1, source: { slug: source.slug, ats: source.ats, token: source.token }, fetchedAt: partition.fetchedAt, jobs: partition.jobs };
-    const response = await fetcher(target, {
-      method: "POST",
-      headers: { "content-type": "application/json", "content-encoding": "gzip" },
-      body: Bun.gzipSync(JSON.stringify(payload)),
-      // Big employers send tens of megabytes and the aggregator ingests one report at a time; give them room.
-      signal: AbortSignal.timeout(options.timeoutMs ?? 120_000),
-    });
-    if (!response.ok) throw new Error(`Aggregator rejected crawl report: HTTP ${response.status}`);
+    const body = Bun.gzipSync(JSON.stringify(payload));
+    let last: Error | undefined;
+    // A crawl that ends while the aggregator is restarting used to lose its whole report in silence.
+    // Retry the failures worth retrying: a refused connection, a timeout, or the server being briefly unwell.
+    for (let attempt = 1; attempt <= attempts; attempt += 1) {
+      try {
+        const response = await fetcher(target, {
+          method: "POST",
+          headers: { "content-type": "application/json", "content-encoding": "gzip" },
+          body,
+          // Big employers send tens of megabytes and the aggregator ingests one report at a time; give them room.
+          signal: AbortSignal.timeout(options.timeoutMs ?? 120_000),
+        });
+        if (response.ok) return;
+        // A rejected report will be rejected again: the catalog disagrees, or the payload is too large.
+        if (response.status < 500) throw new Error(`Aggregator rejected crawl report: HTTP ${response.status}`);
+        last = new Error(`Aggregator returned HTTP ${response.status}`);
+      } catch (error) {
+        if (error instanceof Error && error.message.startsWith("Aggregator rejected")) throw error;
+        last = error instanceof Error ? error : new Error(String(error));
+      }
+      if (attempt < attempts) await sleep(attempt * 5_000);
+    }
+    throw last ?? new Error("Crawl report failed");
   };
 }
 

package/src/index.ts

@@ -19,6 +19,7 @@ export const catalog = createCatalog({ companies });
 export { createCatalog, searchJobs } from "./catalog.ts";
 export { EXPERIENCE_VERSION, experienceLabel, experienceMatches, normalizeJobExperience, statedExperience, titleExperience, type Experience } from "./experience.ts";
 export { matchesSearchTerms, searchHaystack, searchTerms, searchTokens } from "./text-match.ts";
+export { extractSkills } from "./skills.ts";
 export type { Catalog } from "./catalog.ts";
 export type { Ats, Company, CrawlReport, Job, JobPartition, JobSnapshot, JobSummary, SearchQuery } from "./types.ts";
 export { createCrawlReporter, fetchSeedSnapshot, resolveAggregatorUrl } from "./crawl-reporting.ts";

package/src/skills.ts

@@ -0,0 +1,77 @@
+/**
+ * Skill tokens lifted out of a job description so search can find them.
+ *
+ * Search matches a job by its title and employer only, which is why a "react native" search returned 78 India
+ * roles while 171 more carried the term only in their description. Holding the descriptions themselves in the
+ * index is not an option (334 MB across live India and US roles, against 11 MB of titles), so each role keeps
+ * the handful of vocabulary terms its description mentions: a few dozen bytes, and none of the boilerplate.
+ */
+import { searchTokens } from "./text-match.ts";
+
+/**
+ * Terms a candidate actually types. Deliberately narrow: a term earns its place by being a technology or
+ * practice someone searches for, not by appearing often. Two-word entries match only as adjacent words.
+ * Left out on purpose: bare "c", "go" and ".net", which collide with ordinary prose, and employer names such
+ * as Oracle and Workday, which would tag every role at that employer.
+ */
+const VOCABULARY = [
+  // languages
+  "java", "python", "javascript", "typescript", "golang", "ruby", "php", "scala", "kotlin", "swift", "rust",
+  "perl", "matlab", "c++", "c#", "objective c", "dart", "elixir", "haskell", "groovy", "sql", "plsql", "pl sql",
+  // web and frontend
+  "react", "angular", "vue", "svelte", "next.js", "redux", "jquery", "html", "css", "sass", "tailwind",
+  "webpack", "bootstrap", "wordpress", "shopify",
+  // mobile
+  "android", "ios", "react native", "flutter", "swiftui", "jetpack compose", "xcode", "ionic", "cordova", "xamarin",
+  // backend
+  "node.js", "express", "django", "flask", "fastapi", "spring", "spring boot", "hibernate", "laravel", "rails",
+  "asp.net", "graphql", "grpc", "microservices", "kafka", "rabbitmq", "celery", "websocket",
+  // data
+  "mysql", "postgresql", "postgres", "mongodb", "cassandra", "redis", "elasticsearch", "snowflake", "databricks",
+  "hadoop", "spark", "hive", "airflow", "etl", "dbt", "tableau", "power bi", "looker", "bigquery", "redshift",
+  "data warehouse", "data pipeline",
+  // machine learning
+  "machine learning", "deep learning", "tensorflow", "pytorch", "keras", "nlp", "computer vision", "llm",
+  "generative ai", "huggingface", "scikit", "pandas", "numpy", "opencv", "mlops",
+  // cloud and infrastructure
+  "aws", "azure", "gcp", "kubernetes", "docker", "terraform", "ansible", "jenkins", "gitlab", "github",
+  "ci cd", "cicd", "helm", "prometheus", "grafana", "datadog", "splunk", "linux", "nginx", "openshift",
+  "cloudformation", "serverless", "devops", "kibana",
+  // testing
+  "selenium", "cypress", "playwright", "appium", "junit", "testng", "pytest", "jmeter", "postman",
+  "test automation", "automation testing",
+  // security
+  "owasp", "penetration testing", "cryptography", "iam", "soc 2",
+  // embedded and hardware
+  "embedded", "rtos", "verilog", "vhdl", "firmware", "autosar", "fpga", "plc",
+  // enterprise platforms
+  "sap", "abap", "salesforce", "servicenow", "sharepoint", "mulesoft", "sapui5",
+  // practice
+  "agile", "scrum", "jira", "git", "kubernetes", "figma",
+] as const;
+
+/** Vocabulary reduced to the same stemmed tokens the matcher produces, so stored terms and queries agree. */
+const UNIGRAMS = new Set<string>();
+const BIGRAMS = new Set<string>();
+for (const term of VOCABULARY) {
+  const tokens = searchTokens(term);
+  if (tokens.length === 1) UNIGRAMS.add(tokens[0]!);
+  else if (tokens.length === 2) BIGRAMS.add(`${tokens[0]} ${tokens[1]}`);
+}
+
+/**
+ * The vocabulary terms this text mentions, as a space-separated string ready to append to the haystack.
+ * Phrases contribute their own words: matching is term by term, so "react native" is stored as both.
+ */
+export function extractSkills(text: string): string {
+  if (!text) return "";
+  const tokens = searchTokens(text);
+  const hits = new Set<string>();
+  for (let index = 0; index < tokens.length; index += 1) {
+    const token = tokens[index]!;
+    if (UNIGRAMS.has(token)) hits.add(token);
+    const next = tokens[index + 1];
+    if (next !== undefined && BIGRAMS.has(`${token} ${next}`)) { hits.add(token); hits.add(next); }
+  }
+  return [...hits].join(" ");
+}

package/src/types.ts

@@ -102,6 +102,9 @@ export const CASCADE_MINIMUM = 5;
 
 export interface Job extends JobSummary {
   description: string;
+  /** Vocabulary terms read out of the description, so search finds a skill the title never mentions. Set by the
+   * hosted index, which carries these in place of the descriptions themselves; absent means read the description. */
+  skills?: string;
 }
 
 /** Partition lookup that ignores inherited properties, so a slug such as "constructor" never resolves to Object.prototype. */

package/src/version.ts

@@ -1 +1 @@
-export const VERSION = "0.1.45";
+export const VERSION = "0.1.46";