@dxos/nlp 0.9.1-staging.ee54ba693a → 0.11.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@dxos/nlp",
-  "version": "0.9.1-staging.ee54ba693a",
+  "version": "0.11.0",
   "description": "Natural-language parsing: per-word UPOS token structure for intention/transcript analysis.",
   "homepage": "https://dxos.org",
   "bugs": "https://github.com/dxos/dxos/issues",
@@ -16,12 +16,12 @@
     ".": {
       "source": "./src/index.ts",
       "types": "./dist/types/src/index.d.ts",
-      "default": "./dist/lib/neutral/index.mjs"
+      "import": "./dist/lib/index.mjs"
     },
     "./testing": {
       "source": "./src/testing/index.ts",
       "types": "./dist/types/src/testing/index.d.ts",
-      "default": "./dist/lib/neutral/testing/index.mjs"
+      "import": "./dist/lib/testing.mjs"
     }
   },
   "types": "dist/types/src/index.d.ts",
@@ -31,7 +31,7 @@
   ],
   "dependencies": {
     "@effect/ai": "0.36.0",
-    "@dxos/ai": "0.9.1-staging.ee54ba693a"
+    "@dxos/ai": "0.11.0"
   },
   "devDependencies": {
     "effect": "3.21.4"

package/src/align.ts

@@ -22,14 +22,20 @@ export const assembleDocument = (sourceText: string, rawSentences: readonly RawS
       if (start < 0) {
         continue;
       }
+
       const end = start + text.length;
       tokens.push({ index: tokens.length, text, upos, start, end });
       cursor = end;
     }
+
     if (tokens.length > 0) {
       sentences.push({ index: sentenceIndex, start: tokens[0].start, end: tokens[tokens.length - 1].end, tokens });
     }
   });
 
-  return { sourceHash: sourceHash(sourceText), sentences, timestamp: undefined };
+  return {
+    sourceHash: sourceHash(sourceText),
+    sentences,
+    timestamp: undefined,
+  };
 };

package/src/index.ts

@@ -5,4 +5,4 @@
 export * from './Document';
 export * from './align';
 export * from './hash';
-export { type Parser, parseText, stubParse } from './parse';
+export { type Parser, parseText } from './parse';

package/src/parse.test.ts

@@ -5,7 +5,8 @@
 import { describe, test } from 'vitest';
 
 import { assembleDocument } from './align';
-import { type Parser, stubParse } from './parse';
+import { type Parser } from './parse';
+import { stubParse } from './testing';
 
 describe('parser seam', () => {
   test('stubParse satisfies the Parser contract', async ({ expect }) => {

package/src/parse.ts

@@ -10,7 +10,6 @@ import { AiService } from '@dxos/ai';
 
 import { assembleDocument } from './align';
 import { type Document, Upos } from './Document';
-import { stubParse } from './stub';
 
 const PARSE_MODEL = 'com.anthropic.model.claude-haiku-4-5.default';
 
@@ -53,5 +52,3 @@ export const parseText = (text: string) =>
 
 /** The pluggable parser contract consumed by the editor extension and pipeline. */
 export type Parser = (text: string) => Promise<Document>;
-
-export { stubParse };

package/src/testing/index.ts

@@ -4,4 +4,4 @@
 
 // Import directly from `stub` (not `parse`) so the testing entrypoint stays isolated to the
 // offline tagger and does not pull the live-parser/AI stack into story and test bundles.
-export { stubParse } from '../stub';
+export { stubParse } from './stub';

package/src/{stub.ts → testing/stub.ts}

@@ -2,11 +2,11 @@
 // Copyright 2026 DXOS.org
 //
 
-import { assembleDocument } from './align';
-import { type Document, type RawSentence, type Upos } from './Document';
+import { assembleDocument } from '../align';
+import { type Document, type RawSentence, type Upos } from '../Document';
 
-// Closed-class lexicon: small, deterministic, language-is-English assumption (the stub is a demo
-// fallback, not the production tagger). Lowercased keys.
+// Closed-class lexicon: small, deterministic, language-is-English assumption
+// (the stub is a demo fallback, not the production tagger). Lowercased keys.
 const LEXICON: Record<string, Upos> = {
   the: 'DET',
   a: 'DET',
@@ -58,7 +58,9 @@ const LEXICON: Record<string, Upos> = {
 
 const WORD_RE = /[A-Za-z]+(?:'[A-Za-z]+)?|[0-9]+|[.!?,;:]/g;
 
-/** Tag one token by lexicon → number → suffix heuristic → capitalization. `initial` = sentence start. */
+/**
+ * Fake tags one token by lexicon → number → suffix heuristic → capitalization. `initial` = sentence start.
+ */
 const tagWord = (raw: string, initial: boolean): Upos => {
   if (/^[.!?,;:]$/.test(raw)) {
     return 'PUNCT';
@@ -100,9 +102,11 @@ export const stubTag = (text: string): RawSentence[] => {
       initial = true;
     }
   }
+
   if (tokens.length > 0) {
     sentences.push({ tokens });
   }
+
   return sentences;
 };
 

package/dist/lib/neutral/chunk-D4MHDU46.mjs

@@ -1,156 +0,0 @@
-// src/hash.ts
-var sourceHash = (text) => {
-  let hash = 2166136261;
-  for (let index = 0; index < text.length; index++) {
-    hash ^= text.charCodeAt(index);
-    hash = Math.imul(hash, 16777619);
-  }
-  return (hash >>> 0).toString(16).padStart(8, "0");
-};
-
-// src/align.ts
-var assembleDocument = (sourceText, rawSentences) => {
-  let cursor = 0;
-  const sentences = [];
-  rawSentences.forEach((raw, sentenceIndex) => {
-    const tokens = [];
-    for (const { text, upos } of raw.tokens) {
-      const start = sourceText.indexOf(text, cursor);
-      if (start < 0) {
-        continue;
-      }
-      const end = start + text.length;
-      tokens.push({
-        index: tokens.length,
-        text,
-        upos,
-        start,
-        end
-      });
-      cursor = end;
-    }
-    if (tokens.length > 0) {
-      sentences.push({
-        index: sentenceIndex,
-        start: tokens[0].start,
-        end: tokens[tokens.length - 1].end,
-        tokens
-      });
-    }
-  });
-  return {
-    sourceHash: sourceHash(sourceText),
-    sentences,
-    timestamp: void 0
-  };
-};
-
-// src/stub.ts
-var LEXICON = {
-  the: "DET",
-  a: "DET",
-  an: "DET",
-  this: "DET",
-  that: "DET",
-  these: "DET",
-  those: "DET",
-  i: "PRON",
-  you: "PRON",
-  he: "PRON",
-  she: "PRON",
-  it: "PRON",
-  we: "PRON",
-  they: "PRON",
-  is: "AUX",
-  am: "AUX",
-  are: "AUX",
-  was: "AUX",
-  were: "AUX",
-  be: "AUX",
-  been: "AUX",
-  do: "AUX",
-  did: "AUX",
-  in: "ADP",
-  on: "ADP",
-  at: "ADP",
-  of: "ADP",
-  to: "ADP",
-  over: "ADP",
-  under: "ADP",
-  with: "ADP",
-  for: "ADP",
-  and: "CCONJ",
-  or: "CCONJ",
-  but: "CCONJ",
-  because: "SCONJ",
-  if: "SCONJ",
-  while: "SCONJ",
-  although: "SCONJ",
-  not: "PART",
-  very: "ADV",
-  quickly: "ADV",
-  well: "ADV",
-  oh: "INTJ",
-  yes: "INTJ",
-  no: "INTJ"
-};
-var WORD_RE = /[A-Za-z]+(?:'[A-Za-z]+)?|[0-9]+|[.!?,;:]/g;
-var tagWord = (raw, initial) => {
-  if (/^[.!?,;:]$/.test(raw)) {
-    return "PUNCT";
-  }
-  if (/^[0-9]+$/.test(raw)) {
-    return "NUM";
-  }
-  const lower = raw.toLowerCase();
-  if (LEXICON[lower]) {
-    return LEXICON[lower];
-  }
-  if (/^[A-Z]/.test(raw) && !initial) {
-    return "PROPN";
-  }
-  if (/(ing|ed|ize|ise)$/.test(lower)) {
-    return "VERB";
-  }
-  if (/(ly)$/.test(lower)) {
-    return "ADV";
-  }
-  if (/(ous|ful|ive|able|al)$/.test(lower)) {
-    return "ADJ";
-  }
-  return "NOUN";
-};
-var stubTag = (text) => {
-  const sentences = [];
-  let tokens = [];
-  let initial = true;
-  for (const match of text.matchAll(WORD_RE)) {
-    const raw = match[0];
-    tokens.push({
-      text: raw,
-      upos: tagWord(raw, initial)
-    });
-    initial = false;
-    if (/^[.!?]$/.test(raw)) {
-      sentences.push({
-        tokens
-      });
-      tokens = [];
-      initial = true;
-    }
-  }
-  if (tokens.length > 0) {
-    sentences.push({
-      tokens
-    });
-  }
-  return sentences;
-};
-var stubParse = async (text) => assembleDocument(text, stubTag(text));
-
-export {
-  sourceHash,
-  assembleDocument,
-  stubParse
-};
-//# sourceMappingURL=chunk-D4MHDU46.mjs.map

package/dist/lib/neutral/chunk-D4MHDU46.mjs.map

@@ -1,7 +0,0 @@
-{
-  "version": 3,
-  "sources": ["../../../src/hash.ts", "../../../src/align.ts", "../../../src/stub.ts"],
-  "sourcesContent": ["//\n// Copyright 2026 DXOS.org\n//\n\n/**\n * Fast non-cryptographic hash (FNV-1a, 32-bit) of source text. Used purely to detect whether an\n * analyzed span still matches the current editor text — change detection, not security.\n */\nexport const sourceHash = (text: string): string => {\n  let hash = 0x811c9dc5;\n  for (let index = 0; index < text.length; index++) {\n    hash ^= text.charCodeAt(index);\n    hash = Math.imul(hash, 0x01000193);\n  }\n  return (hash >>> 0).toString(16).padStart(8, '0');\n};\n", "//\n// Copyright 2026 DXOS.org\n//\n\nimport { type Document, type RawSentence, type Sentence, type Token } from './Document';\nimport { sourceHash } from './hash';\n\n/**\n * Align offset-free tagger output against the source text to compute exact character offsets.\n * A single forward cursor guarantees repeated surface forms map to successive occurrences rather\n * than re-matching the first. Tokens whose surface form cannot be located ahead of the cursor are\n * dropped (the tagger hallucinated a token), keeping offsets internally consistent.\n */\nexport const assembleDocument = (sourceText: string, rawSentences: readonly RawSentence[]): Document => {\n  let cursor = 0;\n  const sentences: Sentence[] = [];\n\n  rawSentences.forEach((raw, sentenceIndex) => {\n    const tokens: Token[] = [];\n    for (const { text, upos } of raw.tokens) {\n      const start = sourceText.indexOf(text, cursor);\n      if (start < 0) {\n        continue;\n      }\n      const end = start + text.length;\n      tokens.push({ index: tokens.length, text, upos, start, end });\n      cursor = end;\n    }\n    if (tokens.length > 0) {\n      sentences.push({ index: sentenceIndex, start: tokens[0].start, end: tokens[tokens.length - 1].end, tokens });\n    }\n  });\n\n  return { sourceHash: sourceHash(sourceText), sentences, timestamp: undefined };\n};\n", "//\n// Copyright 2026 DXOS.org\n//\n\nimport { assembleDocument } from './align';\nimport { type Document, type RawSentence, type Upos } from './Document';\n\n// Closed-class lexicon: small, deterministic, language-is-English assumption (the stub is a demo\n// fallback, not the production tagger). Lowercased keys.\nconst LEXICON: Record<string, Upos> = {\n  the: 'DET',\n  a: 'DET',\n  an: 'DET',\n  this: 'DET',\n  that: 'DET',\n  these: 'DET',\n  those: 'DET',\n  i: 'PRON',\n  you: 'PRON',\n  he: 'PRON',\n  she: 'PRON',\n  it: 'PRON',\n  we: 'PRON',\n  they: 'PRON',\n  is: 'AUX',\n  am: 'AUX',\n  are: 'AUX',\n  was: 'AUX',\n  were: 'AUX',\n  be: 'AUX',\n  been: 'AUX',\n  do: 'AUX',\n  did: 'AUX',\n  in: 'ADP',\n  on: 'ADP',\n  at: 'ADP',\n  of: 'ADP',\n  to: 'ADP',\n  over: 'ADP',\n  under: 'ADP',\n  with: 'ADP',\n  for: 'ADP',\n  and: 'CCONJ',\n  or: 'CCONJ',\n  but: 'CCONJ',\n  because: 'SCONJ',\n  if: 'SCONJ',\n  while: 'SCONJ',\n  although: 'SCONJ',\n  not: 'PART',\n  very: 'ADV',\n  quickly: 'ADV',\n  well: 'ADV',\n  oh: 'INTJ',\n  yes: 'INTJ',\n  no: 'INTJ',\n};\n\nconst WORD_RE = /[A-Za-z]+(?:'[A-Za-z]+)?|[0-9]+|[.!?,;:]/g;\n\n/** Tag one token by lexicon → number → suffix heuristic → capitalization. `initial` = sentence start. */\nconst tagWord = (raw: string, initial: boolean): Upos => {\n  if (/^[.!?,;:]$/.test(raw)) {\n    return 'PUNCT';\n  }\n  if (/^[0-9]+$/.test(raw)) {\n    return 'NUM';\n  }\n  const lower = raw.toLowerCase();\n  if (LEXICON[lower]) {\n    return LEXICON[lower];\n  }\n  if (/^[A-Z]/.test(raw) && !initial) {\n    return 'PROPN';\n  }\n  if (/(ing|ed|ize|ise)$/.test(lower)) {\n    return 'VERB';\n  }\n  if (/(ly)$/.test(lower)) {\n    return 'ADV';\n  }\n  if (/(ous|ful|ive|able|al)$/.test(lower)) {\n    return 'ADJ';\n  }\n  return 'NOUN';\n};\n\n/** Deterministic UPOS tagger: splits on sentence-final punctuation, tags each token. */\nexport const stubTag = (text: string): RawSentence[] => {\n  const sentences: RawSentence[] = [];\n  let tokens: { text: string; upos: Upos }[] = [];\n  let initial = true;\n  for (const match of text.matchAll(WORD_RE)) {\n    const raw = match[0];\n    tokens.push({ text: raw, upos: tagWord(raw, initial) });\n    initial = false;\n    if (/^[.!?]$/.test(raw)) {\n      sentences.push({ tokens });\n      tokens = [];\n      initial = true;\n    }\n  }\n  if (tokens.length > 0) {\n    sentences.push({ tokens });\n  }\n  return sentences;\n};\n\n/** Parser-shaped wrapper around the stub tagger (async to match the `Parser` contract). */\nexport const stubParse = async (text: string): Promise<Document> => assembleDocument(text, stubTag(text));\n"],
-  "mappings": ";AAQO,IAAMA,aAAa,CAACC,SAAAA;AACzB,MAAIC,OAAO;AACX,WAASC,QAAQ,GAAGA,QAAQF,KAAKG,QAAQD,SAAS;AAChDD,YAAQD,KAAKI,WAAWF,KAAAA;AACxBD,WAAOI,KAAKC,KAAKL,MAAM,QAAA;EACzB;AACA,UAAQA,SAAS,GAAGM,SAAS,EAAA,EAAIC,SAAS,GAAG,GAAA;AAC/C;;;ACFO,IAAMC,mBAAmB,CAACC,YAAoBC,iBAAAA;AACnD,MAAIC,SAAS;AACb,QAAMC,YAAwB,CAAA;AAE9BF,eAAaG,QAAQ,CAACC,KAAKC,kBAAAA;AACzB,UAAMC,SAAkB,CAAA;AACxB,eAAW,EAAEC,MAAMC,KAAI,KAAMJ,IAAIE,QAAQ;AACvC,YAAMG,QAAQV,WAAWW,QAAQH,MAAMN,MAAAA;AACvC,UAAIQ,QAAQ,GAAG;AACb;MACF;AACA,YAAME,MAAMF,QAAQF,KAAKK;AACzBN,aAAOO,KAAK;QAAEC,OAAOR,OAAOM;QAAQL;QAAMC;QAAMC;QAAOE;MAAI,CAAA;AAC3DV,eAASU;IACX;AACA,QAAIL,OAAOM,SAAS,GAAG;AACrBV,gBAAUW,KAAK;QAAEC,OAAOT;QAAeI,OAAOH,OAAO,CAAA,EAAGG;QAAOE,KAAKL,OAAOA,OAAOM,SAAS,CAAA,EAAGD;QAAKL;MAAO,CAAA;IAC5G;EACF,CAAA;AAEA,SAAO;IAAES,YAAYA,WAAWhB,UAAAA;IAAaG;IAAWc,WAAWC;EAAU;AAC/E;;;ACzBA,IAAMC,UAAgC;EACpCC,KAAK;EACLC,GAAG;EACHC,IAAI;EACJC,MAAM;EACNC,MAAM;EACNC,OAAO;EACPC,OAAO;EACPC,GAAG;EACHC,KAAK;EACLC,IAAI;EACJC,KAAK;EACLC,IAAI;EACJC,IAAI;EACJC,MAAM;EACNC,IAAI;EACJC,IAAI;EACJC,KAAK;EACLC,KAAK;EACLC,MAAM;EACNC,IAAI;EACJC,MAAM;EACNC,IAAI;EACJC,KAAK;EACLC,IAAI;EACJC,IAAI;EACJC,IAAI;EACJC,IAAI;EACJC,IAAI;EACJC,MAAM;EACNC,OAAO;EACPC,MAAM;EACNC,KAAK;EACLC,KAAK;EACLC,IAAI;EACJC,KAAK;EACLC,SAAS;EACTC,IAAI;EACJC,OAAO;EACPC,UAAU;EACVC,KAAK;EACLC,MAAM;EACNC,SAAS;EACTC,MAAM;EACNC,IAAI;EACJC,KAAK;EACLC,IAAI;AACN;AAEA,IAAMC,UAAU;AAGhB,IAAMC,UAAU,CAACC,KAAaC,YAAAA;AAC5B,MAAI,aAAaC,KAAKF,GAAAA,GAAM;AAC1B,WAAO;EACT;AACA,MAAI,WAAWE,KAAKF,GAAAA,GAAM;AACxB,WAAO;EACT;AACA,QAAMG,QAAQH,IAAII,YAAW;AAC7B,MAAIrD,QAAQoD,KAAAA,GAAQ;AAClB,WAAOpD,QAAQoD,KAAAA;EACjB;AACA,MAAI,SAASD,KAAKF,GAAAA,KAAQ,CAACC,SAAS;AAClC,WAAO;EACT;AACA,MAAI,oBAAoBC,KAAKC,KAAAA,GAAQ;AACnC,WAAO;EACT;AACA,MAAI,QAAQD,KAAKC,KAAAA,GAAQ;AACvB,WAAO;EACT;AACA,MAAI,yBAAyBD,KAAKC,KAAAA,GAAQ;AACxC,WAAO;EACT;AACA,SAAO;AACT;AAGO,IAAME,UAAU,CAACC,SAAAA;AACtB,QAAMC,YAA2B,CAAA;AACjC,MAAIC,SAAyC,CAAA;AAC7C,MAAIP,UAAU;AACd,aAAWQ,SAASH,KAAKI,SAASZ,OAAAA,GAAU;AAC1C,UAAME,MAAMS,MAAM,CAAA;AAClBD,WAAOG,KAAK;MAAEL,MAAMN;MAAKY,MAAMb,QAAQC,KAAKC,OAAAA;IAAS,CAAA;AACrDA,cAAU;AACV,QAAI,UAAUC,KAAKF,GAAAA,GAAM;AACvBO,gBAAUI,KAAK;QAAEH;MAAO,CAAA;AACxBA,eAAS,CAAA;AACTP,gBAAU;IACZ;EACF;AACA,MAAIO,OAAOK,SAAS,GAAG;AACrBN,cAAUI,KAAK;MAAEH;IAAO,CAAA;EAC1B;AACA,SAAOD;AACT;AAGO,IAAMO,YAAY,OAAOR,SAAoCS,iBAAiBT,MAAMD,QAAQC,IAAAA,CAAAA;",
-  "names": ["sourceHash", "text", "hash", "index", "length", "charCodeAt", "Math", "imul", "toString", "padStart", "assembleDocument", "sourceText", "rawSentences", "cursor", "sentences", "forEach", "raw", "sentenceIndex", "tokens", "text", "upos", "start", "indexOf", "end", "length", "push", "index", "sourceHash", "timestamp", "undefined", "LEXICON", "the", "a", "an", "this", "that", "these", "those", "i", "you", "he", "she", "it", "we", "they", "is", "am", "are", "was", "were", "be", "been", "do", "did", "in", "on", "at", "of", "to", "over", "under", "with", "for", "and", "or", "but", "because", "if", "while", "although", "not", "very", "quickly", "well", "oh", "yes", "no", "WORD_RE", "tagWord", "raw", "initial", "test", "lower", "toLowerCase", "stubTag", "text", "sentences", "tokens", "match", "matchAll", "push", "upos", "length", "stubParse", "assembleDocument"]
-}

package/dist/lib/neutral/index.mjs

@@ -1,78 +0,0 @@
-import {
-  assembleDocument,
-  sourceHash,
-  stubParse
-} from "./chunk-D4MHDU46.mjs";
-
-// src/Document.ts
-import * as Schema from "effect/Schema";
-var Upos = Schema.Literal("ADJ", "ADP", "ADV", "AUX", "CCONJ", "DET", "INTJ", "NOUN", "NUM", "PART", "PRON", "PROPN", "PUNCT", "SCONJ", "SYM", "VERB", "X");
-var Token = Schema.Struct({
-  index: Schema.Number.annotations({
-    description: "Position of the token within its sentence."
-  }),
-  text: Schema.String.annotations({
-    description: "Surface form exactly as it appears in the source."
-  }),
-  upos: Upos.annotations({
-    description: "Universal part-of-speech tag."
-  }),
-  start: Schema.Number,
-  end: Schema.Number
-});
-var Sentence = Schema.Struct({
-  index: Schema.Number,
-  start: Schema.Number,
-  end: Schema.Number,
-  tokens: Schema.Array(Token)
-});
-var Document = Schema.Struct({
-  sourceHash: Schema.String,
-  sentences: Schema.Array(Sentence),
-  timestamp: Schema.optional(Schema.Number)
-});
-
-// src/parse.ts
-import * as LanguageModel from "@effect/ai/LanguageModel";
-import * as Effect from "effect/Effect";
-import * as Schema2 from "effect/Schema";
-import { AiService } from "@dxos/ai";
-var PARSE_MODEL = "com.anthropic.model.claude-haiku-4-5.default";
-var TaggedSentences = Schema2.Struct({
-  sentences: Schema2.Array(Schema2.Struct({
-    tokens: Schema2.Array(Schema2.Struct({
-      text: Schema2.String.annotations({
-        description: "Token surface form exactly as in the source."
-      }),
-      upos: Upos.annotations({
-        description: "Universal POS tag for the token."
-      })
-    }))
-  }))
-});
-var parseText = (text) => Effect.gen(function* () {
-  const { value } = yield* Effect.scoped(LanguageModel.generateObject({
-    schema: TaggedSentences,
-    prompt: [
-      "Tokenize the text below into sentences and tokens, and tag each token with its",
-      "Universal POS tag (UPOS): ADJ ADP ADV AUX CCONJ DET INTJ NOUN NUM PART PRON PROPN",
-      "PUNCT SCONJ SYM VERB X. Return each token surface form exactly as it appears, in order,",
-      "including punctuation as its own PUNCT token. Do not add or omit tokens.",
-      "",
-      "Text:",
-      text
-    ].join("\n")
-  }));
-  return assembleDocument(text, value.sentences);
-}).pipe(Effect.provide(AiService.model(PARSE_MODEL)));
-export {
-  Document,
-  Sentence,
-  Token,
-  Upos,
-  assembleDocument,
-  parseText,
-  sourceHash,
-  stubParse
-};
-//# sourceMappingURL=index.mjs.map

package/dist/lib/neutral/index.mjs.map

@@ -1,7 +0,0 @@
-{
-  "version": 3,
-  "sources": ["../../../src/Document.ts", "../../../src/parse.ts"],
-  "sourcesContent": ["//\n// Copyright 2026 DXOS.org\n//\n\nimport * as Schema from 'effect/Schema';\n\n/** Universal POS tagset (17 tags). https://universaldependencies.org/u/pos/ */\nexport const Upos = Schema.Literal(\n  'ADJ',\n  'ADP',\n  'ADV',\n  'AUX',\n  'CCONJ',\n  'DET',\n  'INTJ',\n  'NOUN',\n  'NUM',\n  'PART',\n  'PRON',\n  'PROPN',\n  'PUNCT',\n  'SCONJ',\n  'SYM',\n  'VERB',\n  'X',\n);\nexport type Upos = Schema.Schema.Type<typeof Upos>;\n\n/** A single word/punctuation token. `start`/`end` are character offsets within the source text. */\nexport const Token = Schema.Struct({\n  index: Schema.Number.annotations({ description: 'Position of the token within its sentence.' }),\n  text: Schema.String.annotations({ description: 'Surface form exactly as it appears in the source.' }),\n  upos: Upos.annotations({ description: 'Universal part-of-speech tag.' }),\n  start: Schema.Number,\n  end: Schema.Number,\n});\nexport type Token = Schema.Schema.Type<typeof Token>;\n\nexport const Sentence = Schema.Struct({\n  index: Schema.Number,\n  start: Schema.Number,\n  end: Schema.Number,\n  tokens: Schema.Array(Token),\n});\nexport type Sentence = Schema.Schema.Type<typeof Sentence>;\n\n/** A parsed document. `sourceHash` is the divergence signal; `timestamp` is debug-only. */\nexport const Document = Schema.Struct({\n  sourceHash: Schema.String,\n  sentences: Schema.Array(Sentence),\n  timestamp: Schema.optional(Schema.Number),\n});\nexport type Document = Schema.Schema.Type<typeof Document>;\n\n/** Raw, offset-free output of a tagger before alignment. */\nexport type RawSentence = { readonly tokens: readonly { readonly text: string; readonly upos: Upos }[] };\n", "//\n// Copyright 2026 DXOS.org\n//\n\nimport * as LanguageModel from '@effect/ai/LanguageModel';\nimport * as Effect from 'effect/Effect';\nimport * as Schema from 'effect/Schema';\n\nimport { AiService } from '@dxos/ai';\n\nimport { assembleDocument } from './align';\nimport { type Document, Upos } from './Document';\nimport { stubParse } from './stub';\n\nconst PARSE_MODEL = 'com.anthropic.model.claude-haiku-4-5.default';\n\n/** LLM output schema: sentences → tokens, no offsets (alignment computes those). */\nconst TaggedSentences = Schema.Struct({\n  sentences: Schema.Array(\n    Schema.Struct({\n      tokens: Schema.Array(\n        Schema.Struct({\n          text: Schema.String.annotations({ description: 'Token surface form exactly as in the source.' }),\n          upos: Upos.annotations({ description: 'Universal POS tag for the token.' }),\n        }),\n      ),\n    }),\n  ),\n});\n\n/**\n * Tag `text` with UPOS via a small LLM, then deterministically align tokens to source offsets.\n * Provides the LanguageModel internally; residual requirement is {@link AiService.AiService}.\n */\nexport const parseText = (text: string) =>\n  Effect.gen(function* () {\n    const { value } = yield* Effect.scoped(\n      LanguageModel.generateObject({\n        schema: TaggedSentences,\n        prompt: [\n          'Tokenize the text below into sentences and tokens, and tag each token with its',\n          'Universal POS tag (UPOS): ADJ ADP ADV AUX CCONJ DET INTJ NOUN NUM PART PRON PROPN',\n          'PUNCT SCONJ SYM VERB X. Return each token surface form exactly as it appears, in order,',\n          'including punctuation as its own PUNCT token. Do not add or omit tokens.',\n          '',\n          'Text:',\n          text,\n        ].join('\\n'),\n      }),\n    );\n    return assembleDocument(text, value.sentences);\n  }).pipe(Effect.provide(AiService.model(PARSE_MODEL)));\n\n/** The pluggable parser contract consumed by the editor extension and pipeline. */\nexport type Parser = (text: string) => Promise<Document>;\n\nexport { stubParse };\n"],
-  "mappings": ";;;;;;;AAIA,YAAYA,YAAY;AAGjB,IAAMC,OAAcC,eACzB,OACA,OACA,OACA,OACA,SACA,OACA,QACA,QACA,OACA,QACA,QACA,SACA,SACA,SACA,OACA,QACA,GAAA;AAKK,IAAMC,QAAeC,cAAO;EACjCC,OAAcC,cAAOC,YAAY;IAAEC,aAAa;EAA6C,CAAA;EAC7FC,MAAaC,cAAOH,YAAY;IAAEC,aAAa;EAAoD,CAAA;EACnGG,MAAMV,KAAKM,YAAY;IAAEC,aAAa;EAAgC,CAAA;EACtEI,OAAcN;EACdO,KAAYP;AACd,CAAA;AAGO,IAAMQ,WAAkBV,cAAO;EACpCC,OAAcC;EACdM,OAAcN;EACdO,KAAYP;EACZS,QAAeC,aAAMb,KAAAA;AACvB,CAAA;AAIO,IAAMc,WAAkBb,cAAO;EACpCc,YAAmBR;EACnBS,WAAkBH,aAAMF,QAAAA;EACxBM,WAAkBC,gBAAgBf,aAAM;AAC1C,CAAA;;;AC/CA,YAAYgB,mBAAmB;AAC/B,YAAYC,YAAY;AACxB,YAAYC,aAAY;AAExB,SAASC,iBAAiB;AAM1B,IAAMC,cAAc;AAGpB,IAAMC,kBAAyBC,eAAO;EACpCC,WAAkBC,cACTF,eAAO;IACZG,QAAeD,cACNF,eAAO;MACZI,MAAaC,eAAOC,YAAY;QAAEC,aAAa;MAA+C,CAAA;MAC9FC,MAAMC,KAAKH,YAAY;QAAEC,aAAa;MAAmC,CAAA;IAC3E,CAAA,CAAA;EAEJ,CAAA,CAAA;AAEJ,CAAA;AAMO,IAAMG,YAAY,CAACN,SACjBO,WAAI,aAAA;AACT,QAAM,EAAEC,MAAK,IAAK,OAAcC,cAChBC,6BAAe;IAC3BC,QAAQhB;IACRiB,QAAQ;MACN;MACA;MACA;MACA;MACA;MACA;MACAZ;MACAa,KAAK,IAAA;EACT,CAAA,CAAA;AAEF,SAAOC,iBAAiBd,MAAMQ,MAAMX,SAAS;AAC/C,CAAA,EAAGkB,KAAYC,eAAQC,UAAUC,MAAMxB,WAAAA,CAAAA,CAAAA;",
-  "names": ["Schema", "Upos", "Literal", "Token", "Struct", "index", "Number", "annotations", "description", "text", "String", "upos", "start", "end", "Sentence", "tokens", "Array", "Document", "sourceHash", "sentences", "timestamp", "optional", "LanguageModel", "Effect", "Schema", "AiService", "PARSE_MODEL", "TaggedSentences", "Struct", "sentences", "Array", "tokens", "text", "String", "annotations", "description", "upos", "Upos", "parseText", "gen", "value", "scoped", "generateObject", "schema", "prompt", "join", "assembleDocument", "pipe", "provide", "AiService", "model"]
-}

package/dist/lib/neutral/meta.json

@@ -1 +0,0 @@
-{"inputs":{"src/Document.ts":{"bytes":4951,"imports":[{"path":"effect/Schema","kind":"import-statement","external":true}],"format":"esm"},"src/hash.ts":{"bytes":1857,"imports":[],"format":"esm"},"src/align.ts":{"bytes":4637,"imports":[{"path":"src/hash.ts","kind":"import-statement","original":"./hash"}],"format":"esm"},"src/stub.ts":{"bytes":9098,"imports":[{"path":"src/align.ts","kind":"import-statement","original":"./align"}],"format":"esm"},"src/parse.ts":{"bytes":6393,"imports":[{"path":"@effect/ai/LanguageModel","kind":"import-statement","external":true},{"path":"effect/Effect","kind":"import-statement","external":true},{"path":"effect/Schema","kind":"import-statement","external":true},{"path":"@dxos/ai","kind":"import-statement","external":true},{"path":"src/align.ts","kind":"import-statement","original":"./align"},{"path":"src/Document.ts","kind":"import-statement","original":"./Document"},{"path":"src/stub.ts","kind":"import-statement","original":"./stub"}],"format":"esm"},"src/index.ts":{"bytes":749,"imports":[{"path":"src/Document.ts","kind":"import-statement","original":"./Document"},{"path":"src/align.ts","kind":"import-statement","original":"./align"},{"path":"src/hash.ts","kind":"import-statement","original":"./hash"},{"path":"src/parse.ts","kind":"import-statement","original":"./parse"}],"format":"esm"},"src/testing/index.ts":{"bytes":892,"imports":[{"path":"src/stub.ts","kind":"import-statement","original":"../stub"}],"format":"esm"}},"outputs":{"dist/lib/neutral/index.mjs.map":{"imports":[],"exports":[],"inputs":{},"bytes":5555},"dist/lib/neutral/index.mjs":{"imports":[{"path":"dist/lib/neutral/chunk-D4MHDU46.mjs","kind":"import-statement"},{"path":"effect/Schema","kind":"import-statement","external":true},{"path":"@effect/ai/LanguageModel","kind":"import-statement","external":true},{"path":"effect/Effect","kind":"import-statement","external":true},{"path":"effect/Schema","kind":"import-statement","external":true},{"path":"@dxos/ai","kind":"import-statement","external":true}],"exports":["Document","Sentence","Token","Upos","assembleDocument","parseText","sourceHash","stubParse"],"entryPoint":"src/index.ts","inputs":{"src/Document.ts":{"bytesInOutput":853},"src/index.ts":{"bytesInOutput":0},"src/parse.ts":{"bytesInOutput":1295}},"bytes":2418},"dist/lib/neutral/testing/index.mjs.map":{"imports":[],"exports":[],"inputs":{},"bytes":93},"dist/lib/neutral/testing/index.mjs":{"imports":[{"path":"dist/lib/neutral/chunk-D4MHDU46.mjs","kind":"import-statement"}],"exports":["stubParse"],"entryPoint":"src/testing/index.ts","inputs":{"src/testing/index.ts":{"bytesInOutput":0}},"bytes":112},"dist/lib/neutral/chunk-D4MHDU46.mjs.map":{"imports":[],"exports":[],"inputs":{},"bytes":7870},"dist/lib/neutral/chunk-D4MHDU46.mjs":{"imports":[],"exports":["assembleDocument","sourceHash","stubParse"],"inputs":{"src/hash.ts":{"bytesInOutput":242},"src/align.ts":{"bytesInOutput":790},"src/stub.ts":{"bytesInOutput":1814}},"bytes":2997}}}

package/dist/lib/neutral/testing/index.mjs

@@ -1,7 +0,0 @@
-import {
-  stubParse
-} from "../chunk-D4MHDU46.mjs";
-export {
-  stubParse
-};
-//# sourceMappingURL=index.mjs.map

package/dist/lib/neutral/testing/index.mjs.map

@@ -1,7 +0,0 @@
-{
-  "version": 3,
-  "sources": [],
-  "sourcesContent": [],
-  "mappings": "",
-  "names": []
-}

package/dist/types/src/stub.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"stub.d.ts","sourceRoot":"","sources":["../../../src/stub.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,KAAK,QAAQ,EAAE,KAAK,WAAW,EAAa,MAAM,YAAY,CAAC;AAkFxE,wFAAwF;AACxF,eAAO,MAAM,OAAO,SAAU,MAAM,KAAG,WAAW,EAkBjD,CAAC;AAEF,2FAA2F;AAC3F,eAAO,MAAM,SAAS,SAAgB,MAAM,KAAG,OAAO,CAAC,QAAQ,CAA0C,CAAC"}

package/dist/types/src/stub.test.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"stub.test.d.ts","sourceRoot":"","sources":["../../../src/stub.test.ts"],"names":[],"mappings":""}