@longtable/research-search 0.1.28

package/README.md

@@ -0,0 +1,11 @@
+# LongTable Search
+
+Scholar-first research search and evidence card workflow for LongTable.
+
+The package converts a research prompt into deterministic query variants, routes
+them through scholarly metadata sources, normalizes results as evidence cards,
+deduplicates overlapping records, and assigns lightweight relevance and citation
+support signals.
+
+It does not scrape paywalled full text, bypass access controls, or claim
+full-paper verification from metadata or abstracts.

package/dist/index.d.ts

@@ -0,0 +1,5 @@
+export * from "./types.js";
+export * from "./query.js";
+export * from "./sources.js";
+export * from "./rank.js";
+export * from "./run.js";

package/dist/index.js

@@ -0,0 +1,5 @@
+export * from "./types.js";
+export * from "./query.js";
+export * from "./sources.js";
+export * from "./rank.js";
+export * from "./run.js";

package/dist/query.d.ts

@@ -0,0 +1,6 @@
+import { type BuildSearchIntentInput, type ResearchSearchIntent, type SearchSource } from "./types.js";
+export declare function normalizeSearchText(value: string): string;
+export declare function splitCsvTerms(value?: string): string[];
+export declare function extractSearchKeywords(text: string, limit?: number): string[];
+export declare function parseSearchSources(value?: string): SearchSource[];
+export declare function buildResearchSearchIntent(input: BuildSearchIntentInput): ResearchSearchIntent;

package/dist/query.js

@@ -0,0 +1,179 @@
+import { SEARCH_SOURCES } from "./types.js";
+const STOP_WORDS = new Set([
+    "a",
+    "about",
+    "an",
+    "and",
+    "are",
+    "as",
+    "at",
+    "be",
+    "before",
+    "between",
+    "but",
+    "by",
+    "can",
+    "do",
+    "does",
+    "for",
+    "from",
+    "has",
+    "have",
+    "how",
+    "in",
+    "into",
+    "is",
+    "it",
+    "its",
+    "of",
+    "on",
+    "or",
+    "should",
+    "study",
+    "that",
+    "the",
+    "their",
+    "this",
+    "to",
+    "what",
+    "when",
+    "where",
+    "whether",
+    "which",
+    "with"
+]);
+function searchId() {
+    return `search_${Date.now().toString(36)}_${Math.random().toString(36).slice(2, 8)}`;
+}
+export function normalizeSearchText(value) {
+    return value
+        .toLowerCase()
+        .replace(/https?:\/\/\S+/g, " ")
+        .replace(/doi:\s*/g, "doi ")
+        .replace(/[^a-z0-9가-힣._:/-]+/gi, " ")
+        .replace(/\s+/g, " ")
+        .trim();
+}
+export function splitCsvTerms(value) {
+    if (!value) {
+        return [];
+    }
+    return value
+        .split(",")
+        .map((entry) => normalizeSearchText(entry))
+        .filter(Boolean);
+}
+export function extractSearchKeywords(text, limit = 12) {
+    const normalized = normalizeSearchText(text);
+    const counts = new Map();
+    for (const token of normalized.split(" ")) {
+        if (token.length < 3 || STOP_WORDS.has(token)) {
+            continue;
+        }
+        counts.set(token, (counts.get(token) ?? 0) + 1);
+    }
+    return [...counts.entries()]
+        .sort((a, b) => {
+        const countDelta = b[1] - a[1];
+        return countDelta !== 0 ? countDelta : a[0].localeCompare(b[0]);
+    })
+        .slice(0, limit)
+        .map(([token]) => token);
+}
+export function parseSearchSources(value) {
+    if (!value || value === "all") {
+        return [...SEARCH_SOURCES];
+    }
+    const requested = value
+        .split(",")
+        .map((entry) => entry.trim())
+        .filter(Boolean);
+    const sources = [];
+    for (const source of requested) {
+        if (!SEARCH_SOURCES.includes(source)) {
+            throw new Error(`Unknown search source: ${source}`);
+        }
+        sources.push(source);
+    }
+    return sources.length > 0 ? sources : [...SEARCH_SOURCES];
+}
+function inferIntentKind(text, explicit) {
+    if (explicit) {
+        if (explicit === "literature" ||
+            explicit === "theory" ||
+            explicit === "measurement" ||
+            explicit === "citation" ||
+            explicit === "metadata" ||
+            explicit === "venue") {
+            return explicit;
+        }
+        throw new Error(`Unknown search intent: ${explicit}`);
+    }
+    const normalized = normalizeSearchText(text);
+    if (/\b(citation|reference|doi|source|hallucination|verify|support)\b|인용|레퍼런스|출처|근거/.test(normalized)) {
+        return "citation";
+    }
+    if (/\b(scale|measure|measurement|instrument|validity|reliability)\b|측정|척도|타당도|도구/.test(normalized)) {
+        return "measurement";
+    }
+    if (/\b(theory|theoretical|framework|construct|conceptual)\b|이론|개념|프레임워크/.test(normalized)) {
+        return "theory";
+    }
+    if (/\b(journal|venue|conference|submission|scope|fit)\b|저널|학회|투고/.test(normalized)) {
+        return "venue";
+    }
+    if (/\b(metadata|pmid|arxiv|openalex|semantic scholar)\b/.test(normalized)) {
+        return "metadata";
+    }
+    return "literature";
+}
+function buildQueryVariants(baseQuery, keywords, field, mustTerms = []) {
+    const variants = new Set();
+    variants.add(baseQuery);
+    const compact = [...mustTerms, ...keywords].slice(0, 8).join(" ");
+    if (compact) {
+        variants.add(compact);
+    }
+    if (field && compact) {
+        variants.add(`${field} ${compact}`);
+    }
+    return [...variants].filter(Boolean).slice(0, 3);
+}
+export function buildResearchSearchIntent(input) {
+    const explicitQuery = input.query?.trim();
+    const baseText = explicitQuery || [
+        input.prompt,
+        input.projectGoal,
+        input.projectBlocker
+    ]
+        .filter((entry) => Boolean(entry && entry.trim()))
+        .join(" ");
+    if (!baseText.trim()) {
+        throw new Error("A search query is required. Pass --query or run inside a workspace with a current goal.");
+    }
+    const mustTerms = splitCsvTerms(input.must);
+    const excludeTerms = splitCsvTerms(input.exclude);
+    const field = input.field?.trim() || undefined;
+    const keywordText = [baseText, field, ...mustTerms].filter(Boolean).join(" ");
+    const keywords = extractSearchKeywords(keywordText);
+    const normalizedQuery = normalizeSearchText(baseText);
+    const query = keywords.length > 0 ? keywords.slice(0, 10).join(" ") : normalizedQuery;
+    const limit = Number.isInteger(input.limit) && input.limit && input.limit > 0
+        ? Math.min(input.limit, 50)
+        : 10;
+    return {
+        id: searchId(),
+        createdAt: new Date().toISOString(),
+        kind: inferIntentKind(baseText, input.intent),
+        query,
+        normalizedQuery,
+        queryVariants: buildQueryVariants(query, keywords, field, mustTerms),
+        keywords,
+        ...(field ? { field } : {}),
+        mustTerms,
+        excludeTerms,
+        requestedSources: parseSearchSources(input.sources),
+        limit,
+        source: input.source ?? "cli"
+    };
+}

package/dist/rank.d.ts

@@ -0,0 +1,2 @@
+import type { EvidenceCard, ResearchSearchIntent } from "./types.js";
+export declare function dedupeAndRankCards(cards: EvidenceCard[], intent: ResearchSearchIntent): EvidenceCard[];

package/dist/rank.js

@@ -0,0 +1,145 @@
+import { normalizeSearchText } from "./query.js";
+function keyForCard(card) {
+    if (card.doi)
+        return `doi:${card.doi.toLowerCase()}`;
+    if (card.pmid)
+        return `pmid:${card.pmid}`;
+    if (card.arxivId)
+        return `arxiv:${card.arxivId.toLowerCase()}`;
+    if (card.openAlexId)
+        return `openalex:${card.openAlexId}`;
+    if (card.semanticScholarId)
+        return `s2:${card.semanticScholarId}`;
+    const title = normalizeSearchText(card.title).replace(/\s+/g, "-");
+    return `title:${title}:${card.year ?? "unknown"}`;
+}
+function keywordMatches(card, keywords) {
+    const haystack = normalizeSearchText([
+        card.title,
+        card.abstract,
+        card.venue,
+        card.researchDesign,
+        card.mainFinding
+    ].filter(Boolean).join(" "));
+    return keywords.filter((keyword) => haystack.includes(normalizeSearchText(keyword)));
+}
+function sourceBoost(card) {
+    if (card.sourceRoutes.includes("openalex"))
+        return 6;
+    if (card.sourceRoutes.includes("semantic_scholar"))
+        return 5;
+    if (card.sourceRoutes.includes("pubmed"))
+        return 5;
+    if (card.sourceRoutes.includes("crossref"))
+        return 4;
+    if (card.sourceRoutes.includes("arxiv"))
+        return 3;
+    if (card.sourceRoutes.includes("doaj"))
+        return 3;
+    if (card.sourceRoutes.includes("eric"))
+        return 3;
+    return 1;
+}
+function supportStatus(card, matches, keywords) {
+    if (!card.abstractAvailable) {
+        return "not_verified";
+    }
+    const ratio = keywords.length === 0 ? 0 : matches.length / keywords.length;
+    if (ratio >= 0.65)
+        return "direct_support";
+    if (ratio >= 0.35)
+        return "indirect_support";
+    if (ratio > 0)
+        return "background";
+    return "questionable_fit";
+}
+function scoreCard(card, intent, matches) {
+    const title = normalizeSearchText(card.title);
+    const abstract = normalizeSearchText(card.abstract ?? "");
+    let score = 0;
+    for (const keyword of intent.keywords) {
+        const normalized = normalizeSearchText(keyword);
+        if (title.includes(normalized))
+            score += 8;
+        if (abstract.includes(normalized))
+            score += 3;
+    }
+    for (const term of intent.mustTerms) {
+        const normalized = normalizeSearchText(term);
+        if (title.includes(normalized) || abstract.includes(normalized)) {
+            score += 10;
+        }
+        else {
+            score -= 20;
+        }
+    }
+    for (const term of intent.excludeTerms) {
+        const normalized = normalizeSearchText(term);
+        if (title.includes(normalized) || abstract.includes(normalized)) {
+            score -= 30;
+        }
+    }
+    if (card.year) {
+        const age = Math.max(0, new Date().getUTCFullYear() - card.year);
+        score += Math.max(0, 10 - Math.min(age, 10));
+    }
+    if (card.citationCount) {
+        score += Math.min(12, Math.log10(card.citationCount + 1) * 4);
+    }
+    if (card.legalFullTextAvailable)
+        score += 4;
+    score += sourceBoost(card);
+    score += matches.length * 2;
+    return Math.max(0, Math.round(score * 10) / 10);
+}
+function mergeCards(existing, incoming) {
+    const sourceRoutes = [...new Set([...existing.sourceRoutes, ...incoming.sourceRoutes])];
+    return {
+        ...existing,
+        authors: existing.authors.length > 0 ? existing.authors : incoming.authors,
+        year: existing.year ?? incoming.year,
+        venue: existing.venue ?? incoming.venue,
+        doi: existing.doi ?? incoming.doi,
+        pmid: existing.pmid ?? incoming.pmid,
+        arxivId: existing.arxivId ?? incoming.arxivId,
+        openAlexId: existing.openAlexId ?? incoming.openAlexId,
+        semanticScholarId: existing.semanticScholarId ?? incoming.semanticScholarId,
+        ericId: existing.ericId ?? incoming.ericId,
+        url: existing.url ?? incoming.url,
+        abstract: existing.abstract ?? incoming.abstract,
+        abstractAvailable: existing.abstractAvailable || incoming.abstractAvailable,
+        legalFullTextAvailable: existing.legalFullTextAvailable || incoming.legalFullTextAvailable,
+        fullTextUrl: existing.fullTextUrl ?? incoming.fullTextUrl,
+        citationCount: Math.max(existing.citationCount ?? 0, incoming.citationCount ?? 0) || undefined,
+        researchDesign: existing.researchDesign ?? incoming.researchDesign,
+        constructsOrMeasures: [...new Set([...(existing.constructsOrMeasures ?? []), ...(incoming.constructsOrMeasures ?? [])])],
+        mainFinding: existing.mainFinding ?? incoming.mainFinding,
+        sourceRoutes,
+        limitations: [...new Set([...existing.limitations, ...incoming.limitations])],
+        matchedKeywords: [...new Set([...existing.matchedKeywords, ...incoming.matchedKeywords])],
+        relevanceScore: Math.max(existing.relevanceScore, incoming.relevanceScore)
+    };
+}
+export function dedupeAndRankCards(cards, intent) {
+    const byKey = new Map();
+    for (const card of cards) {
+        const matches = keywordMatches(card, intent.keywords);
+        const scored = {
+            ...card,
+            matchedKeywords: matches,
+            citationSupportStatus: supportStatus(card, matches, intent.keywords),
+            relevanceScore: scoreCard(card, intent, matches)
+        };
+        const key = keyForCard(scored);
+        const existing = byKey.get(key);
+        byKey.set(key, existing ? mergeCards(existing, scored) : scored);
+    }
+    return [...byKey.values()]
+        .sort((a, b) => {
+        const scoreDelta = b.relevanceScore - a.relevanceScore;
+        if (scoreDelta !== 0)
+            return scoreDelta;
+        return (b.year ?? 0) - (a.year ?? 0);
+    })
+        .slice(0, intent.limit);
+}

package/dist/run.d.ts

@@ -0,0 +1,2 @@
+import type { EvidenceRun, RunResearchSearchInput } from "./types.js";
+export declare function runResearchSearch(input: RunResearchSearchInput): Promise<EvidenceRun>;

package/dist/run.js

@@ -0,0 +1,106 @@
+import { buildResearchSearchIntent } from "./query.js";
+import { dedupeAndRankCards } from "./rank.js";
+import { assessSearchSourceCapabilities, runSourceSearch } from "./sources.js";
+function runId() {
+    return `evidence_${Date.now().toString(36)}_${Math.random().toString(36).slice(2, 8)}`;
+}
+function now() {
+    return new Date().toISOString();
+}
+function defaultFetch() {
+    if (typeof fetch !== "function") {
+        throw new Error("LongTable search requires a fetch-capable Node runtime.");
+    }
+    return fetch;
+}
+export async function runResearchSearch(input) {
+    const createdAt = now();
+    const id = runId();
+    const intent = buildResearchSearchIntent(input);
+    const env = input.env ?? process.env;
+    const capabilities = assessSearchSourceCapabilities(intent.requestedSources, env);
+    const skippedSources = capabilities.filter((capability) => !capability.enabled);
+    if (skippedSources.length > 0 && input.allowPartial !== true) {
+        const updatedAt = now();
+        return {
+            id,
+            createdAt,
+            updatedAt,
+            status: "blocked",
+            intent,
+            sourceReports: skippedSources.map((capability) => ({
+                source: capability.source,
+                status: "skipped",
+                count: 0,
+                elapsedMs: 0,
+                reason: capability.reason
+            })),
+            cards: [],
+            skippedSources,
+            warnings: skippedSources.map((capability) => capability.reason ?? `${capability.source} unavailable.`),
+            blockedReason: "One or more requested scholarly sources are unavailable. Confirm partial search or configure credentials."
+        };
+    }
+    const httpFetch = input.fetch ?? defaultFetch();
+    const sourceReports = [];
+    const cards = [];
+    for (const capability of capabilities) {
+        if (!capability.enabled) {
+            sourceReports.push({
+                source: capability.source,
+                status: "skipped",
+                count: 0,
+                elapsedMs: 0,
+                reason: capability.reason
+            });
+            continue;
+        }
+        const started = Date.now();
+        try {
+            const result = await runSourceSearch({
+                intent,
+                source: capability.source,
+                limit: intent.limit
+            }, {
+                fetch: httpFetch,
+                env
+            });
+            cards.push(...result.cards);
+            sourceReports.push({
+                source: capability.source,
+                status: "completed",
+                count: result.cards.length,
+                elapsedMs: Date.now() - started,
+                endpoint: result.endpoint
+            });
+        }
+        catch (error) {
+            sourceReports.push({
+                source: capability.source,
+                status: "failed",
+                count: 0,
+                elapsedMs: Date.now() - started,
+                reason: error instanceof Error ? error.message : String(error)
+            });
+        }
+    }
+    const rankedCards = dedupeAndRankCards(cards, intent);
+    const hasFailure = sourceReports.some((report) => report.status === "failed" || report.status === "skipped");
+    const status = hasFailure ? "partial" : "completed";
+    return {
+        id,
+        createdAt,
+        updatedAt: now(),
+        status,
+        intent,
+        sourceReports,
+        cards: rankedCards,
+        skippedSources,
+        warnings: [
+            ...skippedSources.map((capability) => capability.reason ?? `${capability.source} unavailable.`),
+            ...sourceReports
+                .filter((report) => report.status === "failed")
+                .map((report) => `${report.source} failed: ${report.reason ?? "unknown error"}`)
+        ]
+    };
+}

package/dist/sources.d.ts

@@ -0,0 +1,5 @@
+import { type SearchSource, type SearchSourceCapability, type SourceSearchContext, type SourceSearchRequest, type SourceSearchResult } from "./types.js";
+export declare function assessSearchSourceCapabilities(sources: SearchSource[], env?: Record<string, string | undefined>): SearchSourceCapability[];
+export declare function enabledSearchSources(sources: SearchSource[], env?: Record<string, string | undefined>): SearchSource[];
+export declare function runSourceSearch(request: SourceSearchRequest, context: SourceSearchContext): Promise<SourceSearchResult>;
+export declare function allSearchSources(): SearchSource[];

package/dist/sources.js

@@ -0,0 +1,510 @@
+import { SEARCH_SOURCES } from "./types.js";
+function endpoint(url, params) {
+    const parsed = new URL(url);
+    for (const [key, value] of Object.entries(params)) {
+        if (value !== undefined && value !== "") {
+            parsed.searchParams.set(key, String(value));
+        }
+    }
+    return parsed.toString();
+}
+function cleanText(value) {
+    if (typeof value !== "string") {
+        return undefined;
+    }
+    const cleaned = value
+        .replace(/<[^>]+>/g, " ")
+        .replace(/&amp;/g, "&")
+        .replace(/&lt;/g, "<")
+        .replace(/&gt;/g, ">")
+        .replace(/&quot;/g, "\"")
+        .replace(/&#39;/g, "'")
+        .replace(/\s+/g, " ")
+        .trim();
+    return cleaned || undefined;
+}
+function asRecord(value) {
+    return value && typeof value === "object" && !Array.isArray(value) ? value : {};
+}
+function asArray(value) {
+    return Array.isArray(value) ? value : [];
+}
+function asString(value) {
+    return typeof value === "string" && value.trim() ? value.trim() : undefined;
+}
+function asNumber(value) {
+    return typeof value === "number" && Number.isFinite(value) ? value : undefined;
+}
+function normalizeDoi(value) {
+    if (!value) {
+        return undefined;
+    }
+    return value
+        .replace(/^https?:\/\/(dx\.)?doi\.org\//i, "")
+        .replace(/^doi:\s*/i, "")
+        .trim()
+        .toLowerCase() || undefined;
+}
+function firstString(value) {
+    const entries = asArray(value);
+    return asString(entries[0]);
+}
+function yearFromParts(value) {
+    const parts = asRecord(value)["date-parts"];
+    const first = asArray(parts)[0];
+    const year = asArray(first)[0];
+    return asNumber(year);
+}
+function inferEvidenceDepth(abstract, legalFullTextAvailable = false) {
+    if (legalFullTextAvailable)
+        return "legal_full_text_available";
+    if (abstract)
+        return "abstract_only";
+    return "metadata_only";
+}
+function inferResearchDesign(abstract) {
+    const normalized = abstract?.toLowerCase() ?? "";
+    if (!normalized)
+        return undefined;
+    if (/\bsystematic review\b|\bmeta-analysis\b/.test(normalized))
+        return "review or meta-analysis";
+    if (/\brandomi[sz]ed\b|\bexperiment\b|\btrial\b/.test(normalized))
+        return "experimental or trial design";
+    if (/\bsurvey\b|\bquestionnaire\b|\bscale\b/.test(normalized))
+        return "survey or scale-based design";
+    if (/\binterview\b|\bqualitative\b|\bcase study\b/.test(normalized))
+        return "qualitative design";
+    if (/\blongitudinal\b|\bpanel data\b/.test(normalized))
+        return "longitudinal design";
+    return undefined;
+}
+function inferMainFinding(abstract) {
+    if (!abstract)
+        return undefined;
+    const sentence = abstract.split(/(?<=[.!?])\s+/).find((entry) => entry.length > 40);
+    return sentence ? sentence.slice(0, 360) : undefined;
+}
+function baseCard(input) {
+    const title = cleanText(input.title) ?? "Untitled scholarly record";
+    const abstract = cleanText(input.abstract);
+    const legalFullTextAvailable = input.legalFullTextAvailable === true;
+    const limitations = [
+        abstract ? "" : "No abstract was available from this source.",
+        legalFullTextAvailable ? "" : "LongTable did not retrieve full text for this card."
+    ].filter(Boolean);
+    return {
+        id: `${input.source}:${input.sourceRecordId ?? normalizeDoi(input.doi) ?? title.toLowerCase().replace(/\s+/g, "-").slice(0, 80)}`,
+        title,
+        authors: input.authors ?? [],
+        year: input.year,
+        venue: cleanText(input.venue),
+        doi: normalizeDoi(input.doi),
+        pmid: input.pmid,
+        arxivId: input.arxivId,
+        openAlexId: input.openAlexId,
+        semanticScholarId: input.semanticScholarId,
+        ericId: input.ericId,
+        url: input.url,
+        sourceRoute: input.source,
+        sourceRoutes: [input.source],
+        sourceRecordId: input.sourceRecordId,
+        abstract,
+        abstractAvailable: Boolean(abstract),
+        evidenceDepth: inferEvidenceDepth(abstract, legalFullTextAvailable),
+        legalFullTextAvailable,
+        fullTextUrl: input.fullTextUrl,
+        citationCount: input.citationCount,
+        researchDesign: inferResearchDesign(abstract),
+        constructsOrMeasures: undefined,
+        mainFinding: inferMainFinding(abstract),
+        relevanceToProject: undefined,
+        citationSupportStatus: "not_verified",
+        limitations,
+        matchedKeywords: [],
+        relevanceScore: 0
+    };
+}
+async function fetchJson(context, url) {
+    const response = await context.fetch(url, {
+        headers: {
+            "accept": "application/json",
+            "user-agent": "LongTable/0.1.28 (https://github.com/HosungYou/LongTable)"
+        }
+    });
+    if (!response.ok) {
+        throw new Error(`HTTP ${response.status} ${response.statusText}`);
+    }
+    return response.json();
+}
+async function fetchText(context, url) {
+    const response = await context.fetch(url, {
+        headers: {
+            "accept": "application/xml, text/xml, application/atom+xml, text/plain",
+            "user-agent": "LongTable/0.1.28 (https://github.com/HosungYou/LongTable)"
+        }
+    });
+    if (!response.ok) {
+        throw new Error(`HTTP ${response.status} ${response.statusText}`);
+    }
+    return response.text();
+}
+function queryForSource(intent) {
+    return intent.queryVariants[0] ?? intent.query;
+}
+function getCapability(source, env) {
+    if (source === "openalex" && !env.OPENALEX_API_KEY) {
+        return {
+            source,
+            enabled: false,
+            requiredEnv: ["OPENALEX_API_KEY"],
+            missingEnv: ["OPENALEX_API_KEY"],
+            reason: "OpenAlex route is disabled because OPENALEX_API_KEY is missing.",
+            setupHint: "Set OPENALEX_API_KEY to enable reliable OpenAlex API use."
+        };
+    }
+    if (source === "unpaywall" && !env.LONGTABLE_CONTACT_EMAIL) {
+        return {
+            source,
+            enabled: false,
+            requiredEnv: ["LONGTABLE_CONTACT_EMAIL"],
+            missingEnv: ["LONGTABLE_CONTACT_EMAIL"],
+            reason: "Unpaywall route is disabled because LONGTABLE_CONTACT_EMAIL is missing.",
+            setupHint: "Set LONGTABLE_CONTACT_EMAIL so Unpaywall can receive the required email parameter."
+        };
+    }
+    return {
+        source,
+        enabled: true,
+        requiredEnv: [],
+        missingEnv: []
+    };
+}
+export function assessSearchSourceCapabilities(sources, env = process.env) {
+    return sources.map((source) => getCapability(source, env));
+}
+export function enabledSearchSources(sources, env = process.env) {
+    return assessSearchSourceCapabilities(sources, env)
+        .filter((capability) => capability.enabled)
+        .map((capability) => capability.source);
+}
+function authorNameFromCrossref(value) {
+    const author = asRecord(value);
+    const given = asString(author.given);
+    const family = asString(author.family);
+    return [given, family].filter(Boolean).join(" ") || asString(author.name);
+}
+async function searchCrossref(request, context) {
+    const url = endpoint("https://api.crossref.org/works", {
+        "query.bibliographic": queryForSource(request.intent),
+        rows: request.limit,
+        mailto: context.env.LONGTABLE_CONTACT_EMAIL
+    });
+    const payload = asRecord(await fetchJson(context, url));
+    const message = asRecord(payload.message);
+    const cards = asArray(message.items).map((item) => {
+        const record = asRecord(item);
+        const authors = asArray(record.author)
+            .map(authorNameFromCrossref)
+            .filter((author) => Boolean(author));
+        return baseCard({
+            source: "crossref",
+            title: firstString(record.title),
+            authors,
+            year: yearFromParts(record.issued),
+            venue: firstString(record["container-title"]),
+            doi: asString(record.DOI),
+            url: asString(record.URL),
+            sourceRecordId: asString(record.DOI),
+            abstract: asString(record.abstract),
+            citationCount: asNumber(record["is-referenced-by-count"])
+        });
+    });
+    return { source: "crossref", endpoint: url, cards };
+}
+function extractXmlBlocks(xml, tag) {
+    const pattern = new RegExp(`<${tag}\\b[^>]*>([\\s\\S]*?)</${tag}>`, "gi");
+    const blocks = [];
+    let match;
+    while ((match = pattern.exec(xml)) !== null) {
+        blocks.push(match[1] ?? "");
+    }
+    return blocks;
+}
+function extractXmlTag(xml, tag) {
+    const escaped = tag.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+    const pattern = new RegExp(`<${escaped}\\b[^>]*>([\\s\\S]*?)</${escaped}>`, "i");
+    return cleanText(pattern.exec(xml)?.[1]);
+}
+async function searchArxiv(request, context) {
+    const url = endpoint("https://export.arxiv.org/api/query", {
+        search_query: `all:${queryForSource(request.intent)}`,
+        start: 0,
+        max_results: request.limit,
+        sortBy: "relevance"
+    });
+    const xml = await fetchText(context, url);
+    const cards = extractXmlBlocks(xml, "entry").map((entry) => {
+        const idUrl = extractXmlTag(entry, "id");
+        const arxivId = idUrl?.split("/abs/")[1];
+        const authors = extractXmlBlocks(entry, "author")
+            .map((author) => extractXmlTag(author, "name"))
+            .filter((author) => Boolean(author));
+        const published = extractXmlTag(entry, "published");
+        return baseCard({
+            source: "arxiv",
+            title: extractXmlTag(entry, "title"),
+            authors,
+            year: published ? Number(published.slice(0, 4)) : undefined,
+            venue: "arXiv",
+            doi: extractXmlTag(entry, "arxiv:doi"),
+            arxivId,
+            url: idUrl,
+            sourceRecordId: arxivId,
+            abstract: extractXmlTag(entry, "summary")
+        });
+    });
+    return { source: "arxiv", endpoint: url, cards };
+}
+async function searchOpenAlex(request, context) {
+    const url = endpoint("https://api.openalex.org/works", {
+        search: queryForSource(request.intent),
+        "per-page": request.limit,
+        api_key: context.env.OPENALEX_API_KEY,
+        mailto: context.env.LONGTABLE_CONTACT_EMAIL
+    });
+    const payload = asRecord(await fetchJson(context, url));
+    const cards = asArray(payload.results).map((item) => {
+        const record = asRecord(item);
+        const primaryLocation = asRecord(record.primary_location);
+        const source = asRecord(primaryLocation.source);
+        const oaLocation = asRecord(record.open_access);
+        const fullTextUrl = asString(primaryLocation.pdf_url) ?? asString(primaryLocation.landing_page_url);
+        const authors = asArray(record.authorships)
+            .map((authorship) => asString(asRecord(asRecord(authorship).author).display_name))
+            .filter((author) => Boolean(author));
+        return baseCard({
+            source: "openalex",
+            title: asString(record.display_name),
+            authors,
+            year: asNumber(record.publication_year),
+            venue: asString(source.display_name),
+            doi: asString(record.doi),
+            openAlexId: asString(record.id),
+            url: asString(record.id),
+            sourceRecordId: asString(record.id),
+            abstract: invertedIndexToAbstract(record.abstract_inverted_index),
+            legalFullTextAvailable: asString(oaLocation.is_oa) === "true" || oaLocation.is_oa === true,
+            fullTextUrl,
+            citationCount: asNumber(record.cited_by_count)
+        });
+    });
+    return { source: "openalex", endpoint: url, cards };
+}
+function invertedIndexToAbstract(value) {
+    const index = asRecord(value);
+    const positions = [];
+    for (const [word, rawPositions] of Object.entries(index)) {
+        for (const position of asArray(rawPositions)) {
+            const numeric = asNumber(position);
+            if (numeric !== undefined) {
+                positions.push([numeric, word]);
+            }
+        }
+    }
+    if (positions.length === 0) {
+        return undefined;
+    }
+    return positions.sort((a, b) => a[0] - b[0]).map(([, word]) => word).join(" ");
+}
+async function searchSemanticScholar(request, context) {
+    const url = endpoint("https://api.semanticscholar.org/graph/v1/paper/research-search", {
+        query: queryForSource(request.intent),
+        limit: request.limit,
+        fields: "title,authors,year,venue,externalIds,abstract,citationCount,openAccessPdf,url"
+    });
+    const headers = {};
+    if (context.env.SEMANTIC_SCHOLAR_API_KEY) {
+        headers["x-api-key"] = context.env.SEMANTIC_SCHOLAR_API_KEY;
+    }
+    const response = await context.fetch(url, {
+        headers: {
+            accept: "application/json",
+            ...headers
+        }
+    });
+    if (!response.ok) {
+        throw new Error(`HTTP ${response.status} ${response.statusText}`);
+    }
+    const payload = asRecord(await response.json());
+    const cards = asArray(payload.data).map((item) => {
+        const record = asRecord(item);
+        const externalIds = asRecord(record.externalIds);
+        const oaPdf = asRecord(record.openAccessPdf);
+        const authors = asArray(record.authors)
+            .map((author) => asString(asRecord(author).name))
+            .filter((author) => Boolean(author));
+        return baseCard({
+            source: "semantic_scholar",
+            title: asString(record.title),
+            authors,
+            year: asNumber(record.year),
+            venue: asString(record.venue),
+            doi: asString(externalIds.DOI),
+            pmid: asString(externalIds.PubMed),
+            arxivId: asString(externalIds.ArXiv),
+            semanticScholarId: asString(record.paperId),
+            url: asString(record.url),
+            sourceRecordId: asString(record.paperId),
+            abstract: asString(record.abstract),
+            legalFullTextAvailable: Boolean(asString(oaPdf.url)),
+            fullTextUrl: asString(oaPdf.url),
+            citationCount: asNumber(record.citationCount)
+        });
+    });
+    return { source: "semantic_scholar", endpoint: url, cards };
+}
+async function searchPubMed(request, context) {
+    const searchUrl = endpoint("https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esearch.fcgi", {
+        db: "pubmed",
+        retmode: "json",
+        term: queryForSource(request.intent),
+        retmax: request.limit,
+        api_key: context.env.NCBI_API_KEY
+    });
+    const searchPayload = asRecord(await fetchJson(context, searchUrl));
+    const ids = asArray(asRecord(searchPayload.esearchresult).idlist)
+        .map(asString)
+        .filter((id) => Boolean(id));
+    if (ids.length === 0) {
+        return { source: "pubmed", endpoint: searchUrl, cards: [] };
+    }
+    const summaryUrl = endpoint("https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esummary.fcgi", {
+        db: "pubmed",
+        retmode: "json",
+        id: ids.join(","),
+        api_key: context.env.NCBI_API_KEY
+    });
+    const summaryPayload = asRecord(await fetchJson(context, summaryUrl));
+    const result = asRecord(summaryPayload.result);
+    const cards = ids.map((id) => {
+        const record = asRecord(result[id]);
+        const authors = asArray(record.authors)
+            .map((author) => asString(asRecord(author).name))
+            .filter((author) => Boolean(author));
+        const pubdate = asString(record.pubdate);
+        return baseCard({
+            source: "pubmed",
+            title: asString(record.title),
+            authors,
+            year: pubdate ? Number(pubdate.slice(0, 4)) : undefined,
+            venue: asString(record.fulljournalname) ?? asString(record.source),
+            pmid: id,
+            url: `https://pubmed.ncbi.nlm.nih.gov/${id}/`,
+            sourceRecordId: id
+        });
+    });
+    return { source: "pubmed", endpoint: summaryUrl, cards };
+}
+async function searchEric(request, context) {
+    const url = endpoint("https://api.ies.ed.gov/eric/", {
+        search: queryForSource(request.intent),
+        format: "json",
+        rows: request.limit
+    });
+    const payload = asRecord(await fetchJson(context, url));
+    const docs = asArray(asRecord(payload.response).docs);
+    const cards = docs.map((item) => {
+        const record = asRecord(item);
+        const id = asString(record.id) ?? asString(record.ericNumber);
+        return baseCard({
+            source: "eric",
+            title: asString(record.title),
+            authors: asArray(record.author).map(asString).filter((author) => Boolean(author)),
+            year: asNumber(record.publicationdateyear) ?? asNumber(record.year),
+            venue: asString(record.source),
+            ericId: id,
+            url: id ? `https://eric.ed.gov/?id=${id}` : undefined,
+            sourceRecordId: id,
+            abstract: asString(record.description) ?? asString(record.abstract)
+        });
+    });
+    return { source: "eric", endpoint: url, cards };
+}
+async function searchDoaj(request, context) {
+    const url = endpoint(`https://doaj.org/api/v4/research-search/articles/${encodeURIComponent(queryForSource(request.intent))}`, {
+        page: 1,
+        pageSize: request.limit
+    });
+    const payload = asRecord(await fetchJson(context, url));
+    const cards = asArray(payload.results).map((item) => {
+        const record = asRecord(item);
+        const bibjson = asRecord(record.bibjson);
+        const journal = asRecord(bibjson.journal);
+        const identifiers = asArray(bibjson.identifier).map(asRecord);
+        const doi = identifiers.find((identifier) => asString(identifier.type)?.toLowerCase() === "doi");
+        const links = asArray(bibjson.link).map(asRecord);
+        const fullText = links.find((link) => asString(link.type)?.toLowerCase().includes("fulltext"));
+        return baseCard({
+            source: "doaj",
+            title: asString(bibjson.title),
+            authors: asArray(bibjson.author).map((author) => asString(asRecord(author).name)).filter((author) => Boolean(author)),
+            year: asNumber(bibjson.year),
+            venue: asString(journal.title),
+            doi: asString(doi?.id),
+            url: asString(record.id) ? `https://doaj.org/article/${record.id}` : undefined,
+            sourceRecordId: asString(record.id),
+            abstract: asString(bibjson.abstract),
+            legalFullTextAvailable: Boolean(fullText),
+            fullTextUrl: asString(fullText?.url)
+        });
+    });
+    return { source: "doaj", endpoint: url, cards };
+}
+async function searchUnpaywall(request, context) {
+    const url = endpoint("https://api.unpaywall.org/v2/research-search/", {
+        query: queryForSource(request.intent),
+        email: context.env.LONGTABLE_CONTACT_EMAIL
+    });
+    const payload = asRecord(await fetchJson(context, url));
+    const cards = asArray(payload.results).map((item) => {
+        const result = asRecord(item);
+        const response = asRecord(result.response);
+        const best = asRecord(response.best_oa_location);
+        return baseCard({
+            source: "unpaywall",
+            title: asString(response.title),
+            authors: [],
+            year: asNumber(response.year),
+            venue: asString(response.journal_name),
+            doi: asString(response.doi),
+            url: asString(response.doi_url),
+            sourceRecordId: asString(response.doi),
+            legalFullTextAvailable: Boolean(asString(best.url)),
+            fullTextUrl: asString(best.url)
+        });
+    });
+    return { source: "unpaywall", endpoint: url, cards };
+}
+export async function runSourceSearch(request, context) {
+    switch (request.source) {
+        case "crossref":
+            return searchCrossref(request, context);
+        case "arxiv":
+            return searchArxiv(request, context);
+        case "openalex":
+            return searchOpenAlex(request, context);
+        case "semantic_scholar":
+            return searchSemanticScholar(request, context);
+        case "pubmed":
+            return searchPubMed(request, context);
+        case "eric":
+            return searchEric(request, context);
+        case "doaj":
+            return searchDoaj(request, context);
+        case "unpaywall":
+            return searchUnpaywall(request, context);
+    }
+}
+export function allSearchSources() {
+    return [...SEARCH_SOURCES];
+}

package/dist/types.d.ts

@@ -0,0 +1,123 @@
+export declare const SEARCH_SOURCES: readonly ["crossref", "arxiv", "openalex", "semantic_scholar", "pubmed", "eric", "doaj", "unpaywall"];
+export type SearchSource = typeof SEARCH_SOURCES[number];
+export type ResearchSearchIntentKind = "literature" | "theory" | "measurement" | "citation" | "metadata" | "venue";
+export type EvidenceDepth = "metadata_only" | "abstract_only" | "legal_full_text_available" | "legal_full_text_unavailable" | "secondary_summary_only";
+export type CitationSupportStatus = "direct_support" | "indirect_support" | "background" | "questionable_fit" | "not_verified";
+export type SourceRunStatus = "completed" | "skipped" | "failed";
+export type EvidenceRunStatus = "completed" | "partial" | "blocked";
+export interface ResearchSearchIntent {
+    id: string;
+    createdAt: string;
+    kind: ResearchSearchIntentKind;
+    query: string;
+    normalizedQuery: string;
+    queryVariants: string[];
+    keywords: string[];
+    field?: string;
+    mustTerms: string[];
+    excludeTerms: string[];
+    requestedSources: SearchSource[];
+    limit: number;
+    source: "cli" | "runtime" | "test";
+}
+export interface EvidenceCard {
+    id: string;
+    title: string;
+    authors: string[];
+    year?: number;
+    venue?: string;
+    doi?: string;
+    pmid?: string;
+    arxivId?: string;
+    openAlexId?: string;
+    semanticScholarId?: string;
+    ericId?: string;
+    url?: string;
+    sourceRoute: SearchSource;
+    sourceRoutes: SearchSource[];
+    sourceRecordId?: string;
+    abstract?: string;
+    abstractAvailable: boolean;
+    evidenceDepth: EvidenceDepth;
+    legalFullTextAvailable: boolean;
+    fullTextUrl?: string;
+    citationCount?: number;
+    researchDesign?: string;
+    constructsOrMeasures?: string[];
+    mainFinding?: string;
+    relevanceToProject?: string;
+    citationSupportStatus: CitationSupportStatus;
+    limitations: string[];
+    matchedKeywords: string[];
+    relevanceScore: number;
+}
+export interface SearchSourceCapability {
+    source: SearchSource;
+    enabled: boolean;
+    requiredEnv: string[];
+    missingEnv: string[];
+    reason?: string;
+    setupHint?: string;
+}
+export interface SourceReport {
+    source: SearchSource;
+    status: SourceRunStatus;
+    count: number;
+    elapsedMs: number;
+    reason?: string;
+    endpoint?: string;
+}
+export interface EvidenceRun {
+    id: string;
+    createdAt: string;
+    updatedAt: string;
+    status: EvidenceRunStatus;
+    intent: ResearchSearchIntent;
+    sourceReports: SourceReport[];
+    cards: EvidenceCard[];
+    skippedSources: SearchSourceCapability[];
+    warnings: string[];
+    blockedReason?: string;
+}
+export interface BuildSearchIntentInput {
+    query?: string;
+    prompt?: string;
+    projectGoal?: string;
+    projectBlocker?: string;
+    intent?: string;
+    field?: string;
+    must?: string;
+    exclude?: string;
+    sources?: string;
+    limit?: number;
+    source?: ResearchSearchIntent["source"];
+}
+export interface HttpResponseLike {
+    ok: boolean;
+    status: number;
+    statusText: string;
+    text(): Promise<string>;
+    json(): Promise<unknown>;
+}
+export type SearchFetch = (url: string, init?: {
+    headers?: Record<string, string>;
+}) => Promise<HttpResponseLike>;
+export interface SourceSearchContext {
+    fetch: SearchFetch;
+    env: Record<string, string | undefined>;
+}
+export interface SourceSearchRequest {
+    intent: ResearchSearchIntent;
+    source: SearchSource;
+    limit: number;
+}
+export interface SourceSearchResult {
+    source: SearchSource;
+    endpoint: string;
+    cards: EvidenceCard[];
+}
+export interface RunResearchSearchInput extends BuildSearchIntentInput {
+    env?: Record<string, string | undefined>;
+    fetch?: SearchFetch;
+    allowPartial?: boolean;
+}

package/dist/types.js

@@ -0,0 +1,10 @@
+export const SEARCH_SOURCES = [
+    "crossref",
+    "arxiv",
+    "openalex",
+    "semantic_scholar",
+    "pubmed",
+    "eric",
+    "doaj",
+    "unpaywall"
+];

package/package.json

@@ -0,0 +1,46 @@
+{
+  "name": "@longtable/research-search",
+  "version": "0.1.28",
+  "private": false,
+  "description": "Scholar-first research search and evidence card workflow for LongTable",
+  "type": "module",
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js"
+    }
+  },
+  "files": [
+    "dist",
+    "README.md"
+  ],
+  "scripts": {
+    "build": "tsc -p tsconfig.json",
+    "typecheck": "tsc -p tsconfig.json --noEmit"
+  },
+  "devDependencies": {
+    "typescript": "^5.6.0"
+  },
+  "keywords": [
+    "longtable",
+    "research",
+    "search",
+    "evidence",
+    "scholarly"
+  ],
+  "author": "Hosung You",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/HosungYou/LongTable.git"
+  },
+  "homepage": "https://github.com/HosungYou/LongTable#readme",
+  "publishConfig": {
+    "access": "public"
+  },
+  "engines": {
+    "node": ">=18.0.0"
+  }
+}