@dxos/nlp 0.0.0

package/package.json

@@ -0,0 +1,46 @@
+{
+  "name": "@dxos/nlp",
+  "version": "0.0.0",
+  "description": "Natural-language parsing: per-word UPOS token structure for intention/transcript analysis.",
+  "homepage": "https://dxos.org",
+  "bugs": "https://github.com/dxos/dxos/issues",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/dxos/dxos"
+  },
+  "license": "FSL-1.1-Apache-2.0",
+  "author": "info@dxos.org",
+  "sideEffects": false,
+  "type": "module",
+  "exports": {
+    ".": {
+      "source": "./src/index.ts",
+      "types": "./dist/types/src/index.d.ts",
+      "default": "./dist/lib/neutral/index.mjs"
+    },
+    "./testing": {
+      "source": "./src/testing/index.ts",
+      "types": "./dist/types/src/testing/index.d.ts",
+      "default": "./dist/lib/neutral/testing/index.mjs"
+    }
+  },
+  "types": "dist/types/src/index.d.ts",
+  "files": [
+    "dist",
+    "src"
+  ],
+  "dependencies": {
+    "@dxos/ai": "workspace:*",
+    "@effect/ai": "catalog:"
+  },
+  "devDependencies": {
+    "effect": "catalog:"
+  },
+  "peerDependencies": {
+    "effect": "catalog:"
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "beast": {}
+}

package/src/Document.ts

@@ -0,0 +1,56 @@
+//
+// Copyright 2026 DXOS.org
+//
+
+import * as Schema from 'effect/Schema';
+
+/** Universal POS tagset (17 tags). https://universaldependencies.org/u/pos/ */
+export const Upos = Schema.Literal(
+  'ADJ',
+  'ADP',
+  'ADV',
+  'AUX',
+  'CCONJ',
+  'DET',
+  'INTJ',
+  'NOUN',
+  'NUM',
+  'PART',
+  'PRON',
+  'PROPN',
+  'PUNCT',
+  'SCONJ',
+  'SYM',
+  'VERB',
+  'X',
+);
+export type Upos = Schema.Schema.Type<typeof Upos>;
+
+/** A single word/punctuation token. `start`/`end` are character offsets within the source text. */
+export const Token = Schema.Struct({
+  index: Schema.Number.annotations({ description: 'Position of the token within its sentence.' }),
+  text: Schema.String.annotations({ description: 'Surface form exactly as it appears in the source.' }),
+  upos: Upos.annotations({ description: 'Universal part-of-speech tag.' }),
+  start: Schema.Number,
+  end: Schema.Number,
+});
+export type Token = Schema.Schema.Type<typeof Token>;
+
+export const Sentence = Schema.Struct({
+  index: Schema.Number,
+  start: Schema.Number,
+  end: Schema.Number,
+  tokens: Schema.Array(Token),
+});
+export type Sentence = Schema.Schema.Type<typeof Sentence>;
+
+/** A parsed document. `sourceHash` is the divergence signal; `timestamp` is debug-only. */
+export const Document = Schema.Struct({
+  sourceHash: Schema.String,
+  sentences: Schema.Array(Sentence),
+  timestamp: Schema.optional(Schema.Number),
+});
+export type Document = Schema.Schema.Type<typeof Document>;
+
+/** Raw, offset-free output of a tagger before alignment. */
+export type RawSentence = { readonly tokens: readonly { readonly text: string; readonly upos: Upos }[] };

package/src/align.test.ts

@@ -0,0 +1,58 @@
+//
+// Copyright 2026 DXOS.org
+//
+
+import { describe, test } from 'vitest';
+
+import { assembleDocument } from './align';
+import { sourceHash } from './hash';
+
+describe('assembleDocument', () => {
+  test('assigns exact offsets to each token', ({ expect }) => {
+    const source = 'The dog barks.';
+    const doc = assembleDocument(source, [
+      {
+        tokens: [
+          { text: 'The', upos: 'DET' },
+          { text: 'dog', upos: 'NOUN' },
+          { text: 'barks', upos: 'VERB' },
+          { text: '.', upos: 'PUNCT' },
+        ],
+      },
+    ]);
+
+    const [sentence] = doc.sentences;
+    expect(sentence.tokens.map((t) => source.slice(t.start, t.end))).toEqual(['The', 'dog', 'barks', '.']);
+    expect(sentence.tokens.map((t) => t.index)).toEqual([0, 1, 2, 3]);
+    expect(sentence.start).toBe(0);
+    expect(sentence.end).toBe(14);
+    expect(doc.sourceHash).toBe(sourceHash(source));
+  });
+
+  test('handles repeated words by scanning forward (no re-match of earlier occurrence)', ({ expect }) => {
+    const source = 'dog dog';
+    const doc = assembleDocument(source, [
+      {
+        tokens: [
+          { text: 'dog', upos: 'NOUN' },
+          { text: 'dog', upos: 'NOUN' },
+        ],
+      },
+    ]);
+    expect(doc.sentences[0].tokens.map((t) => t.start)).toEqual([0, 4]);
+  });
+
+  test('skips tokens not found in source rather than throwing', ({ expect }) => {
+    const source = 'hello world';
+    const doc = assembleDocument(source, [
+      {
+        tokens: [
+          { text: 'hello', upos: 'INTJ' },
+          { text: 'GHOST', upos: 'X' },
+          { text: 'world', upos: 'NOUN' },
+        ],
+      },
+    ]);
+    expect(doc.sentences[0].tokens.map((t) => t.text)).toEqual(['hello', 'world']);
+  });
+});

package/src/align.ts

@@ -0,0 +1,35 @@
+//
+// Copyright 2026 DXOS.org
+//
+
+import { type Document, type RawSentence, type Sentence, type Token } from './Document';
+import { sourceHash } from './hash';
+
+/**
+ * Align offset-free tagger output against the source text to compute exact character offsets.
+ * A single forward cursor guarantees repeated surface forms map to successive occurrences rather
+ * than re-matching the first. Tokens whose surface form cannot be located ahead of the cursor are
+ * dropped (the tagger hallucinated a token), keeping offsets internally consistent.
+ */
+export const assembleDocument = (sourceText: string, rawSentences: readonly RawSentence[]): Document => {
+  let cursor = 0;
+  const sentences: Sentence[] = [];
+
+  rawSentences.forEach((raw, sentenceIndex) => {
+    const tokens: Token[] = [];
+    for (const { text, upos } of raw.tokens) {
+      const start = sourceText.indexOf(text, cursor);
+      if (start < 0) {
+        continue;
+      }
+      const end = start + text.length;
+      tokens.push({ index: tokens.length, text, upos, start, end });
+      cursor = end;
+    }
+    if (tokens.length > 0) {
+      sentences.push({ index: sentenceIndex, start: tokens[0].start, end: tokens[tokens.length - 1].end, tokens });
+    }
+  });
+
+  return { sourceHash: sourceHash(sourceText), sentences, timestamp: undefined };
+};

package/src/hash.test.ts

@@ -0,0 +1,20 @@
+//
+// Copyright 2026 DXOS.org
+//
+
+import { describe, test } from 'vitest';
+
+import { sourceHash } from './hash';
+
+describe('sourceHash', () => {
+  test('returns the expected FNV-1a digest for known inputs', ({ expect }) => {
+    // Pinned so a change to the algorithm or output format is caught; downstream code compares the
+    // stored hash string verbatim across the StateEffect boundary.
+    expect(sourceHash('')).toBe('811c9dc5');
+    expect(sourceHash('a')).toBe('e40c292c');
+  });
+
+  test('is order-sensitive', ({ expect }) => {
+    expect(sourceHash('the lazy dog')).not.toBe(sourceHash('the dog lazy'));
+  });
+});

package/src/hash.ts

@@ -0,0 +1,16 @@
+//
+// Copyright 2026 DXOS.org
+//
+
+/**
+ * Fast non-cryptographic hash (FNV-1a, 32-bit) of source text. Used purely to detect whether an
+ * analyzed span still matches the current editor text — change detection, not security.
+ */
+export const sourceHash = (text: string): string => {
+  let hash = 0x811c9dc5;
+  for (let index = 0; index < text.length; index++) {
+    hash ^= text.charCodeAt(index);
+    hash = Math.imul(hash, 0x01000193);
+  }
+  return (hash >>> 0).toString(16).padStart(8, '0');
+};

package/src/index.ts

@@ -0,0 +1,8 @@
+//
+// Copyright 2026 DXOS.org
+//
+
+export * from './Document';
+export * from './align';
+export * from './hash';
+export { type Parser, parseText, stubParse } from './parse';

package/src/parse.test.ts

@@ -0,0 +1,31 @@
+//
+// Copyright 2026 DXOS.org
+//
+
+import { describe, test } from 'vitest';
+
+import { assembleDocument } from './align';
+import { type Parser, stubParse } from './parse';
+
+describe('parser seam', () => {
+  test('stubParse satisfies the Parser contract', async ({ expect }) => {
+    const parser: Parser = stubParse;
+    const doc = await parser('The cat sleeps.');
+    expect(doc.sentences[0].tokens.length).toBeGreaterThan(0);
+  });
+
+  test('alignment over model-shaped output yields exact offsets', ({ expect }) => {
+    const source = 'Alice runs fast.';
+    const doc = assembleDocument(source, [
+      {
+        tokens: [
+          { text: 'Alice', upos: 'PROPN' },
+          { text: 'runs', upos: 'VERB' },
+          { text: 'fast', upos: 'ADV' },
+          { text: '.', upos: 'PUNCT' },
+        ],
+      },
+    ]);
+    expect(doc.sentences[0].tokens.map((t) => source.slice(t.start, t.end))).toEqual(['Alice', 'runs', 'fast', '.']);
+  });
+});

package/src/parse.ts

@@ -0,0 +1,57 @@
+//
+// Copyright 2026 DXOS.org
+//
+
+import * as LanguageModel from '@effect/ai/LanguageModel';
+import * as Effect from 'effect/Effect';
+import * as Schema from 'effect/Schema';
+
+import { AiService } from '@dxos/ai';
+
+import { assembleDocument } from './align';
+import { type Document, Upos } from './Document';
+import { stubParse } from './stub';
+
+const PARSE_MODEL = 'com.anthropic.model.claude-haiku-4-5.default';
+
+/** LLM output schema: sentences → tokens, no offsets (alignment computes those). */
+const TaggedSentences = Schema.Struct({
+  sentences: Schema.Array(
+    Schema.Struct({
+      tokens: Schema.Array(
+        Schema.Struct({
+          text: Schema.String.annotations({ description: 'Token surface form exactly as in the source.' }),
+          upos: Upos.annotations({ description: 'Universal POS tag for the token.' }),
+        }),
+      ),
+    }),
+  ),
+});
+
+/**
+ * Tag `text` with UPOS via a small LLM, then deterministically align tokens to source offsets.
+ * Provides the LanguageModel internally; residual requirement is {@link AiService.AiService}.
+ */
+export const parseText = (text: string) =>
+  Effect.gen(function* () {
+    const { value } = yield* Effect.scoped(
+      LanguageModel.generateObject({
+        schema: TaggedSentences,
+        prompt: [
+          'Tokenize the text below into sentences and tokens, and tag each token with its',
+          'Universal POS tag (UPOS): ADJ ADP ADV AUX CCONJ DET INTJ NOUN NUM PART PRON PROPN',
+          'PUNCT SCONJ SYM VERB X. Return each token surface form exactly as it appears, in order,',
+          'including punctuation as its own PUNCT token. Do not add or omit tokens.',
+          '',
+          'Text:',
+          text,
+        ].join('\n'),
+      }),
+    );
+    return assembleDocument(text, value.sentences);
+  }).pipe(Effect.provide(AiService.model(PARSE_MODEL)));
+
+/** The pluggable parser contract consumed by the editor extension and pipeline. */
+export type Parser = (text: string) => Promise<Document>;
+
+export { stubParse };

package/src/stub.test.ts

@@ -0,0 +1,31 @@
+//
+// Copyright 2026 DXOS.org
+//
+
+import { describe, test } from 'vitest';
+
+import { stubParse, stubTag } from './stub';
+
+describe('stubTag', () => {
+  test('tags closed-class words from the lexicon and splits sentences', ({ expect }) => {
+    const sentences = stubTag('The dog runs. It barks!');
+    expect(sentences).toHaveLength(2);
+    const first = sentences[0].tokens;
+    expect(first.find((t) => t.text === 'The')?.upos).toBe('DET');
+    expect(first.find((t) => t.text === '.')?.upos).toBe('PUNCT');
+  });
+
+  test('tags capitalized non-initial words as PROPN', ({ expect }) => {
+    const [sentence] = stubTag('I met Alice today');
+    expect(sentence.tokens.find((t) => t.text === 'Alice')?.upos).toBe('PROPN');
+  });
+});
+
+describe('stubParse', () => {
+  test('returns an aligned Document with exact offsets', async ({ expect }) => {
+    const source = 'The dog runs.';
+    const doc = await stubParse(source);
+    expect(doc.sourceHash).toBeTypeOf('string');
+    expect(doc.sentences[0].tokens.map((t) => source.slice(t.start, t.end))).toContain('dog');
+  });
+});

package/src/stub.ts

@@ -0,0 +1,110 @@
+//
+// Copyright 2026 DXOS.org
+//
+
+import { assembleDocument } from './align';
+import { type Document, type RawSentence, type Upos } from './Document';
+
+// Closed-class lexicon: small, deterministic, language-is-English assumption (the stub is a demo
+// fallback, not the production tagger). Lowercased keys.
+const LEXICON: Record<string, Upos> = {
+  the: 'DET',
+  a: 'DET',
+  an: 'DET',
+  this: 'DET',
+  that: 'DET',
+  these: 'DET',
+  those: 'DET',
+  i: 'PRON',
+  you: 'PRON',
+  he: 'PRON',
+  she: 'PRON',
+  it: 'PRON',
+  we: 'PRON',
+  they: 'PRON',
+  is: 'AUX',
+  am: 'AUX',
+  are: 'AUX',
+  was: 'AUX',
+  were: 'AUX',
+  be: 'AUX',
+  been: 'AUX',
+  do: 'AUX',
+  did: 'AUX',
+  in: 'ADP',
+  on: 'ADP',
+  at: 'ADP',
+  of: 'ADP',
+  to: 'ADP',
+  over: 'ADP',
+  under: 'ADP',
+  with: 'ADP',
+  for: 'ADP',
+  and: 'CCONJ',
+  or: 'CCONJ',
+  but: 'CCONJ',
+  because: 'SCONJ',
+  if: 'SCONJ',
+  while: 'SCONJ',
+  although: 'SCONJ',
+  not: 'PART',
+  very: 'ADV',
+  quickly: 'ADV',
+  well: 'ADV',
+  oh: 'INTJ',
+  yes: 'INTJ',
+  no: 'INTJ',
+};
+
+const WORD_RE = /[A-Za-z]+(?:'[A-Za-z]+)?|[0-9]+|[.!?,;:]/g;
+
+/** Tag one token by lexicon → number → suffix heuristic → capitalization. `initial` = sentence start. */
+const tagWord = (raw: string, initial: boolean): Upos => {
+  if (/^[.!?,;:]$/.test(raw)) {
+    return 'PUNCT';
+  }
+  if (/^[0-9]+$/.test(raw)) {
+    return 'NUM';
+  }
+  const lower = raw.toLowerCase();
+  if (LEXICON[lower]) {
+    return LEXICON[lower];
+  }
+  if (/^[A-Z]/.test(raw) && !initial) {
+    return 'PROPN';
+  }
+  if (/(ing|ed|ize|ise)$/.test(lower)) {
+    return 'VERB';
+  }
+  if (/(ly)$/.test(lower)) {
+    return 'ADV';
+  }
+  if (/(ous|ful|ive|able|al)$/.test(lower)) {
+    return 'ADJ';
+  }
+  return 'NOUN';
+};
+
+/** Deterministic UPOS tagger: splits on sentence-final punctuation, tags each token. */
+export const stubTag = (text: string): RawSentence[] => {
+  const sentences: RawSentence[] = [];
+  let tokens: { text: string; upos: Upos }[] = [];
+  let initial = true;
+  for (const match of text.matchAll(WORD_RE)) {
+    const raw = match[0];
+    tokens.push({ text: raw, upos: tagWord(raw, initial) });
+    initial = false;
+    if (/^[.!?]$/.test(raw)) {
+      sentences.push({ tokens });
+      tokens = [];
+      initial = true;
+    }
+  }
+  if (tokens.length > 0) {
+    sentences.push({ tokens });
+  }
+  return sentences;
+};
+
+/** Parser-shaped wrapper around the stub tagger (async to match the `Parser` contract). */
+export const stubParse = async (text: string): Promise<Document> => assembleDocument(text, stubTag(text));

package/src/testing/index.ts

@@ -0,0 +1,7 @@
+//
+// Copyright 2026 DXOS.org
+//
+
+// Import directly from `stub` (not `parse`) so the testing entrypoint stays isolated to the
+// offline tagger and does not pull the live-parser/AI stack into story and test bundles.
+export { stubParse } from '../stub';