@hraness/wordcell 0.25.0 → 0.26.0

package/src/cli-startup.ts

@@ -0,0 +1,76 @@
+import { commandWordCount, resolveCommandId, valueOptions } from "./cli-help.js";
+
+export type StartupCommand =
+  | { readonly kind: "help"; readonly topic?: string }
+  | { readonly kind: "version"; readonly json: boolean }
+  | { readonly kind: "clip"; readonly arguments: readonly string[] }
+  | { readonly kind: "pdf"; readonly arguments: readonly string[] }
+  | { readonly kind: "url-metadata"; readonly arguments: readonly string[] };
+export type StartupParseResult =
+  | { readonly ok: true; readonly value: StartupCommand }
+  | { readonly ok: false; readonly message: string };
+
+function isHelpFlag(argument: string | undefined): boolean {
+  return argument === "--help" || argument === "-h";
+}
+
+/** Help for delegated commands comes from their own parsers. */
+function delegatedHelp(first: string): StartupCommand | undefined {
+  if (first === "clip" || first === "inspect") return { kind: "clip", arguments: ["help"] };
+  if (first === "pdf") return { kind: "pdf", arguments: ["--help"] };
+  if (first === "url-metadata") return { kind: "url-metadata", arguments: ["--help"] };
+  return undefined;
+}
+
+/** `help`, `help advanced`, and `help <command...>`. */
+function parseHelpTopic(words: readonly string[]): StartupParseResult {
+  const topic = words.filter((word) => !isHelpFlag(word) && word !== "--json");
+  if (topic.length === 0) return { ok: true, value: { kind: "help" } };
+  if (topic.length === 1 && topic[0] === "advanced") return { ok: true, value: { kind: "help", topic: "advanced" } };
+  const delegated = delegatedHelp(topic[0] ?? "");
+  if (delegated !== undefined) return { ok: true, value: delegated };
+  const id = resolveCommandId(topic);
+  if (id === undefined) return { ok: false, message: "unknown help topic" };
+  return { ok: true, value: { kind: "help", topic: id } };
+}
+
+/**
+ * `<command> --help` anywhere a person would type it: right after the command
+ * words, or as the last argument when it is not the value of an option.
+ */
+function embeddedHelp(arguments_: readonly string[]): StartupParseResult | undefined {
+  const separator = arguments_.indexOf("--");
+  const end = separator === -1 ? arguments_.length : separator;
+  const index = arguments_.slice(0, end).findIndex(isHelpFlag);
+  if (index <= 0) return undefined;
+  const delegated = delegatedHelp(arguments_[0] ?? "");
+  if (delegated !== undefined) {
+    return index === 1 || index === end - 1 ? { ok: true, value: delegated } : undefined;
+  }
+  const id = resolveCommandId(arguments_.slice(0, index).filter((word) => !word.startsWith("-")));
+  if (id === undefined) return undefined;
+  const words = commandWordCount(id);
+  const previous = arguments_[index - 1] ?? "";
+  // `--help` is never an option value; `-h` could be one, as in `--body -h`.
+  const flag = arguments_[index];
+  const lastArgument = index === end - 1 && (flag === "--help" || !valueOptions(id).has(previous));
+  if (index !== words && !lastArgument) return undefined;
+  return { ok: true, value: { kind: "help", topic: id } };
+}
+
+/** The executable and public parser share the same help/version grammar. */
+export function parseCliStartupCommand(arguments_: readonly string[]): StartupParseResult | undefined {
+  const command = arguments_[0];
+  if (command === undefined) return { ok: true, value: { kind: "help", topic: "start" } };
+  if (command === "help" || isHelpFlag(command)) return parseHelpTopic(arguments_.slice(1));
+  if (command === "--version" || command === "-V" || command === "-v" || command === "version") {
+    return { ok: true, value: { kind: "version", json: arguments_.includes("--json") } };
+  }
+  const help = embeddedHelp(arguments_);
+  if (help !== undefined) return help;
+  if ((command === "clip" || command === "capture" || command === "inspect")
+    && ["--help", "-h", "help"].includes(arguments_[1] ?? "")) {
+    return { ok: true, value: { kind: "clip", arguments: ["help"] } };
+  }
+  return undefined;
+}

package/src/cli-update.ts

@@ -0,0 +1,53 @@
+import { runCliUpdate, type CliUpdateOptions } from "@hraness/cli-update";
+import { readFileSync } from "node:fs";
+import { dirname, join } from "node:path";
+import { findKbPackageRoot } from "./clip/package-root.js";
+import { parseCliStartupCommand } from "./cli-startup.js";
+import { isEvaluationBuilderHelp } from "./evaluation-builder-cli-args.js";
+
+export type WordcellBin = "wordcell" | "wordcell-evaluation-builder";
+
+export function wordcellUpdateOptions(bin: WordcellBin, entrypoint: string, argv: readonly string[]): CliUpdateOptions {
+  const packageRoot = findKbPackageRoot(dirname(entrypoint));
+  const manifest: unknown = JSON.parse(readFileSync(join(packageRoot, "package.json"), "utf8"));
+  if (manifest === null || typeof manifest !== "object" || !("version" in manifest) || typeof manifest.version !== "string") {
+    throw new Error("Wordcell package version is missing.");
+  }
+  const effectFree = bin === "wordcell" ? parseCliStartupCommand(argv) !== undefined : isEvaluationBuilderHelp(argv);
+  return {
+    packageName: "@hraness/wordcell", version: manifest.version, binName: bin, entrypoint,
+    argv: effectFree && argv[0] === "update" ? ["--help"] : argv,
+    provider: { kind: "github", repository: "hraness/wordcell", tagPrefix: "v", assetName: "hraness-wordcell-{version}.tgz" },
+    ignoreScripts: true,
+    effectFree,
+    suppressAutomatic: bin === "wordcell" && argv[0] === "support",
+    async verifyArtifact(artifact) {
+      const { verifyUpdateArtifact } = await import("./cli-release.js");
+      await verifyUpdateArtifact(artifact);
+    },
+  };
+}
+
+/** Admission precedes imports of vault, capture, support, or evaluation code. */
+export async function runWordcellBin(
+  bin: WordcellBin,
+  entrypoint: string,
+  argv: readonly string[],
+  dependencies: { update?: typeof runCliUpdate; run?: () => Promise<number> } = {},
+): Promise<number> {
+  const update = await (dependencies.update ?? runCliUpdate)(wordcellUpdateOptions(bin, entrypoint, argv));
+  if (update.handled) return update.exitCode;
+  try {
+    if (dependencies.run) return await dependencies.run();
+    if (bin === "wordcell") {
+      const { runStandaloneCli } = await import("./cli.js");
+      return await runStandaloneCli(argv);
+    }
+    const { runKbEvidenceRoutingBuildCli, kbEvidenceRoutingBuildUsage } = await import("./evaluation-builder.js");
+    if (isEvaluationBuilderHelp(argv)) console.log(kbEvidenceRoutingBuildUsage);
+    else await runKbEvidenceRoutingBuildCli(argv);
+    return 0;
+  } finally {
+    await update.release();
+  }
+}

package/src/cli.ts

@@ -3,15 +3,16 @@ import { runExecutable } from "./cli-program.js";
 
 export * from "./cli-program.js";
 
-if (import.meta.main) {
+export async function runStandaloneCli(args: readonly string[] = process.argv.slice(2)): Promise<number> {
   const { standaloneSupportEnvironment, isUsefulSupportResult, runProductSupportCommand, showProductSupportInvitation } = await import("./support.js");
   const env = standaloneSupportEnvironment();
-  const args = process.argv.slice(2);
   if (args[0] === "support") {
-    process.exitCode = await runProductSupportCommand(args.slice(1), { env });
+    return await runProductSupportCommand(args.slice(1), { env });
   } else {
     const exitCode = await runExecutable(args);
-    process.exitCode = exitCode;
     if (exitCode === 0 && isUsefulSupportResult(args, env)) await showProductSupportInvitation({ env });
+    return exitCode;
   }
 }
+
+if (import.meta.main) process.exitCode = await runStandaloneCli();

package/src/clip/metadata-search-tool/src/openalex.rs

@@ -170,8 +170,8 @@ fn parse(body: &str, limit: usize, query: &str) -> Outcome {
             Some(cost)
         }
     };
-    let mut seen = std::collections::HashSet::new();
-    let mut seen_ids = std::collections::HashSet::new();
+    let mut known_retracted_ids = std::collections::HashSet::new();
+    let mut observed_ids = Vec::with_capacity(rows.len());
     let mut results = Vec::new();
     for row in rows {
         let id = row
@@ -191,6 +191,18 @@ fn parse(body: &str, limit: usize, query: &str) -> Outcome {
         if title.trim().is_empty() || title.len() > MAX_TEXT {
             return Err(FailureCode::Parse);
         }
+        let title = title.split_whitespace().collect::<Vec<_>>().join(" ");
+        // Absence is unknown; only an explicit flag or title marker excludes a work.
+        let retracted = match row.get("is_retracted") {
+            None => false,
+            Some(value) => value.as_bool().ok_or(FailureCode::Parse)?,
+        };
+        const RETRACTED_PREFIX: &str = "RETRACTED ARTICLE:";
+        let retracted = retracted
+            || title
+                .get(..RETRACTED_PREFIX.len())
+                .is_some_and(|prefix| prefix.eq_ignore_ascii_case(RETRACTED_PREFIX));
+        observed_ids.push(id);
         let mut candidates = Vec::new();
         let mut locations = Vec::new();
         for field in ["best_oa_location", "primary_location"] {
@@ -270,21 +282,41 @@ fn parse(body: &str, limit: usize, query: &str) -> Outcome {
             }
             _ => return Err(FailureCode::Parse),
         }
+        // Validate every row's bounded fields even when its work will be omitted.
+        if retracted {
+            known_retracted_ids.insert(id);
+            continue;
+        }
         // Prefer observed direct destinations over DOI redirectors; preserve stable location order.
         candidates.sort_by_key(|(priority, _)| *priority);
         let Some((_, url)) = candidates.into_iter().next() else {
             continue;
         };
-        if seen_ids.insert(id.to_owned()) && seen.insert(url.clone()) {
-            results.push(SearchResult {
-                title: title.split_whitespace().collect::<Vec<_>>().join(" "),
+        results.push((
+            id,
+            SearchResult {
+                title,
                 url,
                 snippet: Some("Scholarly index metadata; full text not fetched.".into()),
                 source_engine: "openalex".into(),
-            });
-        }
+            },
+        ));
     }
-    if !rows.is_empty() && results.is_empty() {
+    // A duplicate row cannot reintroduce an identity explicitly marked retracted.
+    // Deduplicate only admitted works so excluded identities cannot hide usable rows.
+    let mut seen = std::collections::HashSet::new();
+    let mut seen_ids = std::collections::HashSet::new();
+    let results: Vec<_> = results
+        .into_iter()
+        .filter(|(id, _)| !known_retracted_ids.contains(id))
+        .filter(|(id, result)| seen_ids.insert(*id) && seen.insert(result.url.clone()))
+        .map(|(_, result)| result)
+        .collect();
+    if results.is_empty()
+        && observed_ids
+            .iter()
+            .any(|id| !known_retracted_ids.contains(id))
+    {
         return Err(FailureCode::UnverifiedEmpty);
     }
     Ok(Discovery {
@@ -307,6 +339,122 @@ mod tests {
         parse(&value.to_string(), 3, "microalgae")
     }
     #[test]
+    fn omits_known_retractions_and_preserves_cost_for_policy_empty_results() {
+        let mut retracted = row();
+        retracted["is_retracted"] = json!(true);
+        let mut value = response(vec![retracted.clone()]);
+        value["meta"]["cost_usd"] = json!(0.0001);
+        let result = check(value).unwrap();
+        assert!(result.results.is_empty());
+        assert_eq!(result.reported_cost_usd, Some(0.0001));
+
+        let mut good = row();
+        good["id"] = json!("https://openalex.org/W456");
+        good["is_retracted"] = json!(false);
+        let result = check(response(vec![retracted, good])).unwrap();
+        assert_eq!(result.results.len(), 1);
+        assert_eq!(
+            result.results[0].title,
+            "Microalgae harvesting energy efficiency"
+        );
+        // Missing status remains admissible metadata, without a verified-status claim.
+        let result = check(response(vec![row()])).unwrap();
+        assert_eq!(result.results.len(), 1);
+        assert_eq!(
+            result.results[0].snippet.as_deref(),
+            Some("Scholarly index metadata; full text not fetched.")
+        );
+    }
+    #[test]
+    fn retraction_marker_is_anchored_and_does_not_exclude_scholarship_about_retractions() {
+        for title in [
+            "RETRACTED ARTICLE: Microalgae",
+            " retracted article:Microalgae",
+            "RETRACTED\n ARTICLE: Microalgae",
+        ] {
+            let mut r = row();
+            r["title"] = json!(title);
+            r["is_retracted"] = json!(false);
+            assert!(
+                check(response(vec![r])).unwrap().results.is_empty(),
+                "{title}"
+            );
+        }
+        for title in [
+            "A study of retractions",
+            "Analysis of RETRACTED ARTICLE: labels",
+            "RETRACTED ARTICLES: a review",
+            "Retraction notices in science",
+            "論文について",
+        ] {
+            let mut r = row();
+            r["title"] = json!(title);
+            assert_eq!(
+                check(response(vec![r])).unwrap().results.len(),
+                1,
+                "{title}"
+            );
+        }
+    }
+    #[test]
+    fn rejects_malformed_status_even_with_an_explicit_title_marker() {
+        for status in [Value::Null, json!("true"), json!(1), json!([]), json!({})] {
+            for title in ["Microalgae harvesting", "RETRACTED ARTICLE: Microalgae"] {
+                let mut r = row();
+                r["title"] = json!(title);
+                r["is_retracted"] = status.clone();
+                assert_eq!(check(response(vec![r])).unwrap_err(), FailureCode::Parse);
+            }
+        }
+    }
+    #[test]
+    fn policy_filter_preserves_unusable_link_failures_and_row_validation() {
+        let mut retracted = row();
+        retracted["is_retracted"] = json!(true);
+        let mut unusable = row();
+        unusable["id"] = json!("https://openalex.org/W456");
+        unusable["best_oa_location"] = Value::Null;
+        unusable["doi"] = Value::Null;
+        assert_eq!(
+            check(response(vec![retracted.clone(), unusable])).unwrap_err(),
+            FailureCode::UnverifiedEmpty
+        );
+        retracted["doi"] = json!(42);
+        assert_eq!(
+            check(response(vec![retracted.clone()])).unwrap_err(),
+            FailureCode::Parse
+        );
+        retracted["doi"] = Value::Null;
+        retracted["locations"] = json!([{"is_oa":true,"landing_page_url":"x".repeat(8193)}]);
+        assert_eq!(
+            check(response(vec![retracted])).unwrap_err(),
+            FailureCode::BodyLimit
+        );
+    }
+    #[test]
+    fn duplicate_work_cannot_reintroduce_a_known_retraction_in_either_order() {
+        let mut retracted = row();
+        retracted["is_retracted"] = json!(true);
+        for rows in [vec![row(), retracted.clone()], vec![retracted, row()]] {
+            assert!(check(response(rows)).unwrap().results.is_empty());
+        }
+    }
+    #[test]
+    fn excluded_identity_cannot_consume_a_valid_works_url_deduplication_slot() {
+        let mut retracted = row();
+        retracted["is_retracted"] = json!(true);
+        let mut good = row();
+        good["id"] = json!("https://openalex.org/W456");
+        good["title"] = json!("Allowed work at the shared destination");
+        let result = check(response(vec![row(), good, retracted])).unwrap();
+        assert_eq!(result.results.len(), 1);
+        assert_eq!(
+            result.results[0].title,
+            "Allowed work at the shared destination"
+        );
+        assert_eq!(result.results[0].url, "https://repository.example/paper");
+    }
+    #[test]
     fn discovery_prefers_observed_oa_landing_and_deduplicates() {
         let results = check(response(vec![row(), row()])).unwrap().results;
         assert_eq!(results.len(), 1);

package/src/evaluation-builder-bin.ts

@@ -0,0 +1,5 @@
+#!/usr/bin/env bun
+import { fileURLToPath } from "node:url";
+import { runWordcellBin } from "./cli-update.js";
+
+if (import.meta.main) process.exitCode = await runWordcellBin("wordcell-evaluation-builder", fileURLToPath(import.meta.url), process.argv.slice(2));

package/src/evaluation-builder-cli-args.ts

@@ -0,0 +1,30 @@
+export const kbEvidenceRoutingBuildUsage =
+  "Usage: wordcell-evaluation-builder <--anchor-seal|--build> --config <checked-config.json> --artifact-root <artifact-B>\n"
+  + "       wordcell-evaluation-builder update [check|status|enable|disable] [--json]";
+
+export function parseKbEvidenceRoutingBuildCliArguments(arguments_: readonly string[]): {
+  readonly mode: "anchor-seal" | "build";
+  readonly configPath: string;
+  readonly artifactRoot: string;
+} {
+  const mode = arguments_[0];
+  const configPath = arguments_[2];
+  const artifactRoot = arguments_[4];
+  if (
+    arguments_.length !== 5
+    || (mode !== "--anchor-seal" && mode !== "--build")
+    || arguments_[1] !== "--config"
+    || arguments_[3] !== "--artifact-root"
+    || configPath === undefined
+    || configPath.trim() === ""
+    || artifactRoot === undefined
+    || artifactRoot.trim() === ""
+  ) {
+    throw new Error(kbEvidenceRoutingBuildUsage);
+  }
+  return Object.freeze({ mode: mode.slice(2) as "anchor-seal" | "build", configPath, artifactRoot });
+}
+
+export function isEvaluationBuilderHelp(arguments_: readonly string[]): boolean {
+  return arguments_.length === 1 && (arguments_[0] === "--help" || arguments_[0] === "-h");
+}

package/src/evaluation-builder.ts

@@ -1,4 +1,6 @@
 #!/usr/bin/env bun
+import { isEvaluationBuilderHelp, kbEvidenceRoutingBuildUsage, parseKbEvidenceRoutingBuildCliArguments } from "./evaluation-builder-cli-args.js";
+export { kbEvidenceRoutingBuildUsage, parseKbEvidenceRoutingBuildCliArguments } from "./evaluation-builder-cli-args.js";
 
 import { createHash, randomBytes } from "node:crypto";
 import { constants as fsConstants } from "node:fs";
@@ -2459,32 +2461,6 @@ export async function buildKbEvidenceRoutingEvaluation(
   };
 }
 
-export const kbEvidenceRoutingBuildUsage =
-  "Usage: wordcell-evaluation-builder <--anchor-seal|--build> --config <checked-config.json> --artifact-root <artifact-B>";
-
-export function parseKbEvidenceRoutingBuildCliArguments(arguments_: readonly string[]): {
-  readonly mode: "anchor-seal" | "build";
-  readonly configPath: string;
-  readonly artifactRoot: string;
-} {
-  const mode = arguments_[0];
-  const configPath = arguments_[2];
-  const artifactRoot = arguments_[4];
-  if (
-    arguments_.length !== 5
-    || (mode !== "--anchor-seal" && mode !== "--build")
-    || arguments_[1] !== "--config"
-    || arguments_[3] !== "--artifact-root"
-    || configPath === undefined
-    || configPath.trim() === ""
-    || artifactRoot === undefined
-    || artifactRoot.trim() === ""
-  ) {
-    throw new Error(kbEvidenceRoutingBuildUsage);
-  }
-  return Object.freeze({ mode: mode.slice(2) as "anchor-seal" | "build", configPath, artifactRoot });
-}
-
 export async function runKbEvidenceRoutingBuildCli(
   arguments_: readonly string[],
   dependencies: Pick<KbEvidenceRoutingEvaluationBuildDependencies, "runGit"> = {},
@@ -2518,7 +2494,7 @@ export async function runKbEvidenceRoutingBuildCli(
 
 if (import.meta.main) {
   const arguments_ = process.argv.slice(2);
-  if (arguments_.length === 1 && (arguments_[0] === "--help" || arguments_[0] === "-h")) {
+  if (isEvaluationBuilderHelp(arguments_)) {
     console.log(kbEvidenceRoutingBuildUsage);
   } else {
     await runKbEvidenceRoutingBuildCli(arguments_);

package/src/import-supermemory.ts

@@ -828,7 +828,9 @@ export function occupancyKey(value: string): string {
 
 /** Validate `--prefix` as a vault-relative directory note-ID prefix. */
 export function normalizePrefix(prefix: string): string {
-  const trimmed = prefix.replace(/\/+$/u, "");
+  let end = prefix.length;
+  while (end > 0 && prefix.charCodeAt(end - 1) === 47) end -= 1;
+  const trimmed = prefix.slice(0, end);
   let canonical: string | undefined;
   try {
     canonical = canonicalNoteId(trimmed);

package/dist/{index-hxyjer3x.js → index-hacpnaew.js}

@@ -1,13 +1,13 @@
 // @bun
+import {
+  sanitizeArtifactUrl
+} from "./index-mxxxytys.js";
 import {
   assertSafeNetworkUrl,
   decodeBytes,
   isPrivateHostname,
   safeFetch
 } from "./index-e5fbsywq.js";
-import {
-  sanitizeArtifactUrl
-} from "./index-mxxxytys.js";
 
 // src/clip/url-intelligence.ts
 var MAX_METADATA_SEARCH_RESULTS = 20;

package/dist/{index-qfqjfxaa.js → index-m9p2a4mj.js}

@@ -11,16 +11,16 @@ import {
   terminalOutput,
   yamlString
 } from "./index-cd75vky9.js";
+import {
+  redactSensitiveText,
+  sanitizeArtifactUrl
+} from "./index-mxxxytys.js";
 import {
   safeFetch
 } from "./index-e5fbsywq.js";
 import {
   BoundedByteBuffer
 } from "./index-gh719d91.js";
-import {
-  redactSensitiveText,
-  sanitizeArtifactUrl
-} from "./index-mxxxytys.js";
 import {
   sanitizeTerminalLine,
   sanitizeTerminalText

package/dist/{index-xwxy71ew.js → index-ncwzmsge.js}

@@ -8,6 +8,9 @@ import {
   readCookieFile,
   renderCookieHeader
 } from "./index-2gv8y733.js";
+import {
+  sanitizeArtifactUrl
+} from "./index-mxxxytys.js";
 import {
   startNetworkProxy
 } from "./index-w2zc0vwa.js";
@@ -16,15 +19,12 @@ import {
   decodeBytes,
   safeFetch
 } from "./index-e5fbsywq.js";
-import {
-  captureUrl
-} from "./index-byz4kzww.js";
 import {
   readBoundedByteStream
 } from "./index-gh719d91.js";
 import {
-  sanitizeArtifactUrl
-} from "./index-mxxxytys.js";
+  captureUrl
+} from "./index-byz4kzww.js";
 import {
   findKbPackageRoot,
   resolvePackageDirectory

package/dist/{index-e26mdwxz.js → index-wqx1x32f.js}

@@ -1,7 +1,4 @@
 // @bun
-import {
-  percolateVault
-} from "./index-bdwcjvr4.js";
 import {
   createGraphSnapshot,
   openGraphAuthority
@@ -10,6 +7,9 @@ import {
   GraphAuthorityError,
   validateGraphPercolationOptions
 } from "./index-pgtm2nhf.js";
+import {
+  percolateVault
+} from "./index-bdwcjvr4.js";
 import {
   analyzeVault,
   lookupNote,