neuralos 3.9.6 → 3.10.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -54,7 +54,72 @@ neuralos info
54
54
 
55
55
  ## Platforms
56
56
 
57
- Bundled engines: macOS arm64/x64, Linux x64/arm64 (glibc), Windows x64.
57
+ ## Structured extraction (same API as Python)
58
+
59
+ Declare the record, hand over messy text, get typed fields back. The API is the
60
+ Node mirror of the Python `needle.extract` — same semantics, same validation:
61
+
62
+ ```js
63
+ const { extract, Field } = require('neuralos');
64
+
65
+ const invoice = extract('Invoice INV-1042 issued 12 May 2024, total 49.62 USD.', {
66
+ name: 'Invoice', // named schema = the engine's call target
67
+ parameters: {
68
+ type: 'object',
69
+ properties: {
70
+ invoice_no: Field({ type: 'string' }),
71
+ issue_date: Field({ type: 'string', format: 'date' }),
72
+ total: Field({ type: 'number', ge: 0 }),
73
+ },
74
+ required: ['invoice_no', 'total'],
75
+ },
76
+ });
77
+ // { invoice_no: 'INV-1042', issue_date: '2024-05-12', total: 49.62 }
78
+ ```
79
+
80
+ With `strict: true` (the default) a value the input does not license raises
81
+ `ExtractionValidationError` instead of being returned silently — a date whose
82
+ year is not written in the text, an invented number, or a negated request:
83
+
84
+ ```js
85
+ const { extract, ExtractionValidationError } = require('neuralos');
86
+ try {
87
+ extract('Invoice with no date mentioned', { name: 'Invoice', parameters: { /* … */ } });
88
+ } catch (err) {
89
+ if (err instanceof ExtractionValidationError) console.error(err.message);
90
+ }
91
+ ```
92
+
93
+ Return value: the extracted record, or `null` when the model emits no call.
94
+ Pass `strict: false` to receive unvalidated values.
95
+
96
+ ### CLI
97
+
98
+ ```bash
99
+ neuralos extract --schema invoice.schema.json --text "Invoice INV-7, 3 March 2024, total 12.50"
100
+ neuralos extract --schema invoice.schema.json --text "…" --no-strict
101
+ ```
102
+
103
+ Exits `1` with the grounding message when strict validation fails.
104
+
105
+ ## Feature parity with the Python distribution
106
+
107
+ | Capability | Python (`pip install neuralos`) | Node (this package) |
108
+ |---|---|---|
109
+ | Tool calling | `Needle().run()` | `new Needle().run()` |
110
+ | Structured extraction | `needle.extract()` | `extract()` |
111
+ | Field constraints | `needle.Field(...)` | `Field(...)` |
112
+ | Grounding validation | `ExtractionValidationError` | `ExtractionValidationError` |
113
+ | Embeddings | `.embed()` | `.embed()` |
114
+ | CLI | `run \| embed \| info` | `run \| extract \| embed \| info` |
115
+ | Engine + weights bundled | wheels | platform packages |
116
+
117
+ Both distributions are released **at the same version number** from the same
118
+ tag, so `npm view neuralos version` and `pip index versions neuralos` agree.
119
+
120
+ Bundled engines: macOS arm64/x64, Linux x64/arm64 (glibc), **Windows x64**.
121
+ Other platforms are not packaged — the Python distribution
122
+ (`pip install neuralos`) carries a broader engine matrix.
58
123
  Other platforms are not packaged — the Python distribution
59
124
  (`pip install neuralos`) carries a broader engine matrix.
60
125
 
package/cli.js CHANGED
@@ -3,19 +3,23 @@
3
3
 
4
4
  const fs = require('fs');
5
5
  const path = require('path');
6
- const { Needle } = require('./index');
6
+ const { Needle, extract, ExtractionValidationError } = require('./index');
7
7
 
8
8
  function usage(code = 0) {
9
9
  const text = `neuralos ${require('./package.json').version} — offline tool-calling + embeddings engine
10
10
 
11
11
  Usage:
12
- neuralos run --tools <tools.json> [--impl <impl.js>] [--system <text>]
13
- [--query <text>] [--max-steps <n>] [--max-new-tokens <n>]
14
- neuralos embed --text <text>
12
+ neuralos run --tools <tools.json> [--impl <impl.js>] [--system <text>]
13
+ [--query <text>] [--max-steps <n>] [--max-new-tokens <n>]
14
+ neuralos extract --schema <schema.json> --text <text> [--system <text>]
15
+ [--max-new-tokens <n>] [--no-strict]
16
+ neuralos embed --text <text>
15
17
  neuralos info
16
18
 
17
- tools.json: array of { name, description, parameters }
18
- impl.js: optional module exporting { toolName: (args) => result } functions
19
+ tools.json: array of { name, description, parameters }
20
+ impl.js: optional module exporting { toolName: (args) => result } functions
21
+ schema.json: a JSON schema object (the record is the only tool). Prints the
22
+ extracted record, or exits 1 with the grounding failure.
19
23
  `;
20
24
  process.stdout.write(text);
21
25
  process.exit(code);
@@ -38,6 +42,27 @@ try {
38
42
  process.exit(0);
39
43
  }
40
44
 
45
+ if (command === 'extract') {
46
+ const schemaPath = argValue('--schema', argv);
47
+ const text = argValue('--text', argv);
48
+ if (!schemaPath || !text) usage(1);
49
+ const schema = JSON.parse(fs.readFileSync(path.resolve(schemaPath), 'utf8'));
50
+ const system = argValue('--system', argv) || '';
51
+ const maxNewTokens = Number(argValue('--max-new-tokens', argv) || 512);
52
+ const strict = !argv.includes('--no-strict');
53
+ try {
54
+ const record = extract(text, schema, { system, maxNewTokens, strict });
55
+ process.stdout.write(JSON.stringify(record, null, 2) + '\n');
56
+ process.exit(0);
57
+ } catch (err) {
58
+ if (err instanceof ExtractionValidationError) {
59
+ process.stderr.write(`neuralos: ${err.message}\n`);
60
+ process.exit(1);
61
+ }
62
+ throw err;
63
+ }
64
+ }
65
+
41
66
  if (command === 'embed') {
42
67
  const text = argValue('--text', argv);
43
68
  if (!text) usage(1);
package/index.js CHANGED
@@ -17,10 +17,25 @@
17
17
  *
18
18
  * const agent = new Needle({ tools: [getWeather] });
19
19
  * console.log(agent.run("What's the weather in Lagos?"));
20
+ *
21
+ * Structured extraction (Python parity):
22
+ *
23
+ * const { extract, Field } = require('neuralos');
24
+ * const invoice = extract('Invoice INV-7, 12 May 2024, total 49.62', {
25
+ * type: 'object',
26
+ * properties: {
27
+ * invoice_no: Field({ type: 'string' }),
28
+ * issue_date: Field({ type: 'string', format: 'date' }),
29
+ * total: Field({ type: 'number', ge: 0 }),
30
+ * },
31
+ * required: ['invoice_no', 'total'],
32
+ * });
20
33
  */
21
34
 
22
35
  const engineLoader = require('./engine');
23
- const { normalizeTools, tool } = require('./tools');
36
+ const { normalizeTools, tool, Field } = require('./tools');
37
+ const { ExtractionValidationError, sourceYears, groundedNumberPaths,
38
+ ungroundedPaths, annotateUngrounded, validateExtraction } = require('./validate');
24
39
 
25
40
  const UNRESET_TURNS = 4;
26
41
 
@@ -52,6 +67,7 @@ class Needle {
52
67
  this.systemText = withDateFact(system, autoDate);
53
68
  this.toolsJson = JSON.stringify(this._schemas);
54
69
  this._stateless = !!stateless;
70
+ this._seenYears = new Set();
55
71
  this._turns = 0;
56
72
  this._bufferSize = bufferSize;
57
73
  this._bind();
@@ -76,7 +92,7 @@ class Needle {
76
92
  return this._complete(text, maxNewTokens);
77
93
  }
78
94
 
79
- _complete(text, maxNewTokens, parse = true) {
95
+ _complete(text, maxNewTokens, parse = true, annotate = true) {
80
96
  this._bind();
81
97
  const e = engineLoader.engine();
82
98
  const buffer = e.allocBuffer(this._bufferSize);
@@ -93,11 +109,15 @@ class Needle {
93
109
  } catch (err) {
94
110
  throw new Error(`neuralOS: engine returned an unparseable envelope (${err.message})`);
95
111
  }
112
+ for (const year of sourceYears(text || '')) this._seenYears.add(year);
113
+ if (annotate) {
114
+ annotateUngrounded(response, this._schemas, this._seenYears, this.systemText, text);
115
+ }
96
116
  return response;
97
117
  }
98
118
 
99
119
  /** Execute tool calls until the model responds. Mirrors Python's run(). */
100
- run(query = '', { maxSteps = 8, maxNewTokens = 512 } = {}) {
120
+ run(query = '', { maxSteps = 8, maxNewTokens = 512, strict = true } = {}) {
101
121
  if (this._stateless) this.reset();
102
122
  this._countQuery();
103
123
  let response = this._complete(query, maxNewTokens);
@@ -106,8 +126,18 @@ class Needle {
106
126
  const calls = response.function_calls || [];
107
127
  if (response.type !== 'call' || !calls.length) break;
108
128
  const results = [];
129
+ const ungrounded = ungroundedPaths(response);
109
130
  for (const call of calls) {
110
131
  const name = String(call.name);
132
+ let fabricated = [...(ungrounded.get(name) || [])].sort();
133
+ if (strict && fabricated.length) {
134
+ const grounded = groundedNumberPaths(call.arguments || {}, query, this.systemText);
135
+ fabricated = fabricated.filter((path) => !grounded.has(path));
136
+ }
137
+ if (strict && fabricated.length) {
138
+ results.push({ error: 'ungrounded ' + fabricated.join(', ') });
139
+ continue;
140
+ }
111
141
  const fn = this._functions.get(name);
112
142
  if (!fn) {
113
143
  results.push({ error: 'unknown tool: ' + name });
@@ -120,7 +150,7 @@ class Needle {
120
150
  }
121
151
  }
122
152
  executed.push(...results);
123
- response = this._complete(JSON.stringify(results), maxNewTokens, false);
153
+ response = this._complete(JSON.stringify(results), maxNewTokens, false, false);
124
154
  try { response = JSON.parse(response); } catch (_) { /* handled below */ }
125
155
  if (typeof response !== 'object' || response === null) {
126
156
  throw new Error('neuralOS: engine returned an unparseable envelope');
@@ -141,16 +171,56 @@ class Needle {
141
171
  return e.decodeFloats(out, dim);
142
172
  }
143
173
 
174
+ /** One-shot structured extraction against this agent's system prompt. */
175
+ extract(text, schema, { maxNewTokens = 512, strict = true } = {}) {
176
+ return extract(text, schema, {
177
+ system: this.systemText, maxNewTokens, strict, autoDate: false,
178
+ });
179
+ }
180
+
144
181
  reset() {
145
182
  this._bind();
146
183
  engineLoader.engine().reset();
147
184
  this._turns = 0;
185
+ this._seenYears = new Set();
186
+ }
187
+
188
+ close() {
189
+ this._closed = true;
190
+ }
191
+ }
192
+
193
+ /**
194
+ * One-shot structured extraction — Python-parity semantics:
195
+ * the record is the only tool, the first call is the answer, and with
196
+ * strict=true (default) ungrounded values raise ExtractionValidationError
197
+ * instead of being returned silently.
198
+ */
199
+ function extract(text, schema, { system = '', maxNewTokens = 512, strict = true,
200
+ autoDate = true } = {}) {
201
+ const agent = new Needle({ tools: [schema], system, autoDate });
202
+ let response;
203
+ try {
204
+ response = agent._complete(text, maxNewTokens);
205
+ } finally {
206
+ agent.close();
148
207
  }
208
+ const calls = (response.function_calls && response.function_calls.length)
209
+ ? response.function_calls
210
+ : (response.suppressed_calls || []);
211
+ if (!calls.length) return null;
212
+ const arguments_ = calls[0].arguments || {};
213
+ if (strict) validateExtraction(text, schema, arguments_, response, agent.systemText);
214
+ return arguments_;
149
215
  }
150
216
 
151
217
  module.exports = {
152
218
  Needle,
153
219
  tool,
220
+ Field,
221
+ extract,
222
+ ExtractionValidationError,
223
+ validate: require('./validate'),
154
224
  version: require('./package.json').version,
155
225
  platformKey: engineLoader.platformKey,
156
226
  engineInfo: () => {
package/package.json CHANGED
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "name": "neuralos",
3
- "version": "3.9.6",
4
- "description": "neuralOS - automation foundation model for tiny devices: offline tool-calling + embeddings engine for Node.js, weights bundled",
3
+ "version": "3.10.0",
4
+ "description": "neuralOS - automation foundation model for tiny devices: offline tool-calling, structured extraction and embeddings for Node.js, weights bundled",
5
5
  "main": "index.js",
6
6
  "bin": {
7
7
  "neuralos": "cli.js"
@@ -9,6 +9,7 @@
9
9
  "files": [
10
10
  "index.js",
11
11
  "engine.js",
12
+ "validate.js",
12
13
  "tools.js",
13
14
  "cli.js",
14
15
  "test.js",
@@ -40,10 +41,10 @@
40
41
  "koffi": "^2.9.0"
41
42
  },
42
43
  "optionalDependencies": {
43
- "neuralos-darwin-arm64": "3.9.6",
44
- "neuralos-darwin-x64": "3.9.6",
45
- "neuralos-linux-x64-gnu": "3.9.6",
46
- "neuralos-linux-arm64-gnu": "3.9.6",
47
- "neuralos-win32-x64": "3.9.6"
44
+ "neuralos-darwin-arm64": "3.10.0",
45
+ "neuralos-darwin-x64": "3.10.0",
46
+ "neuralos-linux-x64-gnu": "3.10.0",
47
+ "neuralos-linux-arm64-gnu": "3.10.0",
48
+ "neuralos-win32-x64": "3.10.0"
48
49
  }
49
50
  }
package/tools.js CHANGED
@@ -28,6 +28,11 @@ function normalizeTools(tools, functions) {
28
28
  if (typeof entry.implementation === 'function') {
29
29
  functions.set(entry.name, entry.implementation);
30
30
  }
31
+ } else if (entry && typeof entry === 'object') {
32
+ // An ANONYMOUS schema: the record itself is the only tool (Python parity —
33
+ // needle._resolve passes plain dicts through unwrapped, which is what
34
+ // extract() relies on: the engine answers with arguments only).
35
+ schemas.push(entry);
31
36
  } else {
32
37
  throw new TypeError(
33
38
  'neuralOS: each tool must be a schema object {name, description, parameters, implementation?} ' +
@@ -44,4 +49,41 @@ function tool(schema, fn) {
44
49
  return wrapped;
45
50
  }
46
51
 
47
- module.exports = { normalizeTools, tool };
52
+ /**
53
+ * Field — declarative constraints for a schema property, mirroring the Python
54
+ * `needle.Field`. Build the property, then attach it to your JSON schema:
55
+ *
56
+ * parameters: {
57
+ * type: 'object',
58
+ * properties: {
59
+ * total: Field({ type: 'number', description: 'invoice total', ge: 0 }),
60
+ * status: Field({ type: 'string', enum: ['paid', 'unpaid', 'overdue'] }),
61
+ * },
62
+ * required: ['total'],
63
+ * }
64
+ *
65
+ * Python parity: ge/le → minimum/maximum, gt/lt → exclusiveMinimum/Maximum,
66
+ * min_length/max_length → minLength/maxLength, min_items/max_items →
67
+ * minItems/maxItems, plus enum/const/pattern/format/multipleOf.
68
+ */
69
+ function Field({ default: defaultValue, description, enum: enum_, const: const_,
70
+ ge, le, gt, lt, multiple_of: multipleOf, min_length: minLength,
71
+ max_length: maxLength, pattern, format, min_items: minItems,
72
+ max_items: maxItems, unique_items: uniqueItems, ...rest } = {}) {
73
+ const schema = { ...rest };
74
+ const pairs = [['description', description], ['enum', enum_],
75
+ ['minimum', ge], ['maximum', le],
76
+ ['exclusiveMinimum', gt], ['exclusiveMaximum', lt],
77
+ ['multipleOf', multipleOf], ['minLength', minLength],
78
+ ['maxLength', maxLength], ['pattern', pattern], ['format', format],
79
+ ['minItems', minItems], ['maxItems', maxItems],
80
+ ['uniqueItems', uniqueItems]];
81
+ for (const [key, value] of pairs) {
82
+ if (value !== undefined && value !== null) schema[key] = value;
83
+ }
84
+ if (const_ !== undefined) schema.const = const_;
85
+ if (defaultValue !== undefined) schema.default = defaultValue;
86
+ return schema;
87
+ }
88
+
89
+ module.exports = { normalizeTools, tool, Field };
package/validate.js ADDED
@@ -0,0 +1,263 @@
1
+ /**
2
+ * Grounding + extraction validation — the Node port of the Python
3
+ * `needle` validation core (needle/__init__.py), kept behaviour-compatible:
4
+ *
5
+ * * sourceYears() — years literally written in the input
6
+ * * licensedYears() — those years, plus the system date's year when the
7
+ * input reasons relatively ("tomorrow", "next week")
8
+ * * walkGrounding() — date arguments must carry a licensed year
9
+ * * groundedNumberPaths() — numbers a value may carry without being invented
10
+ * * annotateUngrounded() — the engine's own fabrication report, re-checked
11
+ * * validateExtraction() — strict mode: raise instead of returning
12
+ * ungrounded values silently
13
+ *
14
+ * Python raises ExtractionValidationError; so do we.
15
+ */
16
+ 'use strict';
17
+
18
+ class ExtractionValidationError extends Error {
19
+ constructor(message) {
20
+ super(message);
21
+ this.name = 'ExtractionValidationError';
22
+ }
23
+ }
24
+
25
+ const MONTHS =
26
+ '(?:jan(?:uary)?|feb(?:ruary)?|mar(?:ch)?|apr(?:il)?|may|' +
27
+ 'jun(?:e)?|jul(?:y)?|aug(?:ust)?|sep(?:t(?:ember)?)?|' +
28
+ 'oct(?:ober)?|nov(?:ember)?|dec(?:ember)?)';
29
+
30
+ // The number after a day+month is its year, unless it is the hour of a time
31
+ // ("5 June 19:30", "June 5 7 pm"): no ISO argument can carry that as a year,
32
+ // so licensing it would reject every date. Year-first numeric dates only
33
+ // (2024-03-15) — a day/month-first date must not mint its leading component.
34
+ const SOURCE_YEAR_PATTERNS = [
35
+ new RegExp(`\\b\\d{1,2}(?:st|nd|rd|th)?\\s+${MONTHS}[\\s,]+(\\d{1,4})(?![0-9A-Za-z]|:\\d|\\s*[ap]\\.?m\\b)`, 'g'),
36
+ new RegExp(`\\b${MONTHS}\\s+\\d{1,2}(?:st|nd|rd|th)?\\s*,?\\s+(\\d{1,4})(?![0-9A-Za-z]|:\\d|\\s*[ap]\\.?m\\b)`, 'g'),
37
+ new RegExp(`\\b${MONTHS}[\\s,]+(\\d{3,4})(?![0-9A-Za-z])`, 'g'),
38
+ /\byear\s+(\d{1,4})(?![0-9A-Za-z])/g,
39
+ /(?<![0-9])(\d{4})(?=[-/]\d{1,2}[-/]\d{1,2}(?![0-9]))/g,
40
+ ];
41
+
42
+ const RELATIVE_CUE = new RegExp(
43
+ '\\b(today|tonight|tomorrow|yesterday|next|this|coming|now|in \\d+ (?:days?|weeks?|months?|years?)|' +
44
+ 'monday|tuesday|wednesday|thursday|friday|saturday|sunday)\\b', 'i');
45
+
46
+ const NUMBER_TOKEN =
47
+ /(?<![\w.,])(?:[-+]?\d{1,3}(?:,\d{3})+(?:\.\d+)?|[-+]?\d+(?:\.\d+)?)(?!\d)/g;
48
+
49
+ const DATE_FACT =
50
+ /date:\s*\d{4}-\d{2}-\d{2}(?:\s+[A-Za-z]{3})?(?:\s+\d{2}:\d{2})?|\d{4}-\d{2}-\d{2}T\d{2}:\d{2}(?::\d{2})?/g;
51
+
52
+ /** Years literally written in the text. */
53
+ function sourceYears(text) {
54
+ const lowered = String(text || '').toLowerCase();
55
+ const years = new Set();
56
+ for (const pattern of SOURCE_YEAR_PATTERNS) {
57
+ pattern.lastIndex = 0;
58
+ let match;
59
+ while ((match = pattern.exec(lowered)) !== null) {
60
+ years.add(parseInt(match[1], 10));
61
+ }
62
+ }
63
+ return years;
64
+ }
65
+
66
+ function relativeCue(text) {
67
+ return !!text && RELATIVE_CUE.test(String(text));
68
+ }
69
+
70
+ /** Years a date argument may carry without being fabricated. */
71
+ function licensedYears(seenYears, system, relative = true) {
72
+ const years = new Set(seenYears || []);
73
+ if (years.size && system && relative) {
74
+ for (const year of sourceYears(system)) years.add(year);
75
+ }
76
+ return years;
77
+ }
78
+
79
+ function schemaParameters(schema) {
80
+ if (schema && typeof schema === 'object' && schema.parameters) return schema.parameters;
81
+ return schema || {};
82
+ }
83
+
84
+ function resolveRef(node, root) {
85
+ const seen = new Set();
86
+ let current = node;
87
+ while (current && typeof current === 'object' && current.$ref) {
88
+ if (seen.has(current.$ref)) break;
89
+ seen.add(current.$ref);
90
+ const parts = String(current.$ref).replace(/^#\//, '').split('/');
91
+ let target = root;
92
+ for (const part of parts) {
93
+ target = (target || {})[part.replace(/~1/g, '/').replace(/~0/g, '~')] || {};
94
+ }
95
+ if (target === current) break;
96
+ current = target;
97
+ }
98
+ return current;
99
+ }
100
+
101
+ /** Date arguments must carry a year that is licensed by the input. */
102
+ function walkGrounding(schema, arguments_, years) {
103
+ const root = schemaParameters(schema);
104
+ const checked = new Set();
105
+ const failures = new Set();
106
+
107
+ const walk = (value, node, path) => {
108
+ node = resolveRef(node, root) || {};
109
+ const variants = node.anyOf || node.oneOf || [];
110
+ const concrete = variants.filter((v) => (resolveRef(v, root) || {}).type !== 'null');
111
+ if (concrete.length === 1) node = resolveRef(concrete[0], root) || {};
112
+
113
+ if ((node.format === 'date' || node.format === 'date-time') && typeof value === 'string') {
114
+ const match = /^(\d{4})-/.exec(value);
115
+ if (match && years.size) {
116
+ checked.add(path);
117
+ if (!years.has(parseInt(match[1], 10))) failures.add(path);
118
+ }
119
+ return;
120
+ }
121
+ if (value && typeof value === 'object' && !Array.isArray(value)) {
122
+ const properties = node.properties || {};
123
+ for (const [key, item] of Object.entries(value)) {
124
+ if (key in properties) walk(item, properties[key], path ? `${path}.${key}` : key);
125
+ }
126
+ } else if (Array.isArray(value) && node.items) {
127
+ value.forEach((item, index) => walk(item, node.items, `${path}[${index}]`));
128
+ }
129
+ };
130
+
131
+ walk(arguments_, root, '');
132
+ return { checked, failures };
133
+ }
134
+
135
+ function temporalGrounding(text, schema, arguments_, system) {
136
+ const years = licensedYears(sourceYears(text), system, relativeCue(text));
137
+ return walkGrounding(schema, arguments_, years);
138
+ }
139
+
140
+ function withoutDateFacts(text) {
141
+ return String(text || '').replace(DATE_FACT, ' ');
142
+ }
143
+
144
+ /** (path, value) for every numeric leaf, arguments-relative. */
145
+ function numericPaths(arguments_, path = '') {
146
+ const out = [];
147
+ const walk = (value, currentPath) => {
148
+ if (typeof value === 'boolean') return;
149
+ if (typeof value === 'number' && Number.isFinite(value)) {
150
+ out.push([currentPath, value]);
151
+ return;
152
+ }
153
+ if (Array.isArray(value)) {
154
+ value.forEach((item, index) => walk(item, `${currentPath}[${index}]`));
155
+ } else if (value && typeof value === 'object') {
156
+ for (const [key, item] of Object.entries(value)) {
157
+ walk(item, currentPath ? `${currentPath}.${key}` : String(key));
158
+ }
159
+ }
160
+ };
161
+ walk(arguments_, path);
162
+ return out;
163
+ }
164
+
165
+ /** Numeric values written in the source texts (separators normalized). */
166
+ function sourceNumbers(...sources) {
167
+ const text = sources
168
+ .map((source) => withoutDateFacts(source))
169
+ .filter((source) => source)
170
+ .join('\n');
171
+ const numbers = new Set();
172
+ NUMBER_TOKEN.lastIndex = 0;
173
+ let match;
174
+ while ((match = NUMBER_TOKEN.exec(text)) !== null) {
175
+ const normalized = match[0].replace(/,/g, '');
176
+ const asNumber = Number(normalized);
177
+ if (Number.isFinite(asNumber)) numbers.add(asNumber);
178
+ }
179
+ return numbers;
180
+ }
181
+
182
+ function groundedNumberPaths(arguments_, ...sources) {
183
+ const numbers = sourceNumbers(...sources);
184
+ const grounded = new Set();
185
+ if (!numbers.size) return grounded;
186
+ for (const [path, value] of numericPaths(arguments_)) {
187
+ if (numbers.has(value)) grounded.add(path);
188
+ }
189
+ return grounded;
190
+ }
191
+
192
+ /** The engine's fabrication report, grouped by tool. */
193
+ function ungroundedPaths(response) {
194
+ const validation = (response && response.validation) || {};
195
+ const grouped = new Map();
196
+ for (const name of validation.ungrounded || []) {
197
+ const text = String(name);
198
+ const dot = text.indexOf('.');
199
+ const tool = dot === -1 ? text : text.slice(0, dot);
200
+ const path = dot === -1 ? text : text.slice(dot + 1);
201
+ if (!grouped.has(tool)) grouped.set(tool, new Set());
202
+ grouped.get(tool).add(path || tool);
203
+ }
204
+ return grouped;
205
+ }
206
+
207
+ /** Add engine-reported ungrounded values to response.validation.ungrounded. */
208
+ function annotateUngrounded(response, toolSchemas, seenYears, system, text) {
209
+ const calls = response.function_calls || [];
210
+ const years = licensedYears(seenYears, system, text !== undefined && text !== null ? relativeCue(text) : true);
211
+ if (!calls.length || !years.size) return;
212
+ const schemas = new Map();
213
+ for (const entry of toolSchemas || []) {
214
+ if (entry && entry.name) schemas.set(entry.name, entry);
215
+ }
216
+ const found = [];
217
+ for (const call of calls) {
218
+ const schema = schemas.get(call.name);
219
+ if (!schema) continue;
220
+ const { failures } = walkGrounding(schema, call.arguments || {}, years);
221
+ for (const path of [...failures].sort()) found.push(`${call.name}.${path}`);
222
+ }
223
+ if (!found.length) return;
224
+ const validation = response.validation || {};
225
+ const existing = [...(validation.ungrounded || [])];
226
+ validation.ungrounded = existing.concat(found.filter((name) => !existing.includes(name)));
227
+ response.validation = validation;
228
+ }
229
+
230
+ /** Strict mode: raise instead of returning values the input does not license. */
231
+ function validateExtraction(text, schema, arguments_, response, system) {
232
+ const { checked, failures } = temporalGrounding(text, schema, arguments_, system);
233
+ const validation = (response && response.validation) || {};
234
+ const flagged = validation.ungrounded || [];
235
+ const grounded = flagged.length ? groundedNumberPaths(arguments_, text, system) : new Set();
236
+ for (const name of flagged) {
237
+ const path = String(name).includes('.') ? String(name).split('.').slice(1).join('.') : String(name);
238
+ if (grounded.has(path)) continue;
239
+ if (!checked.has(path) || failures.has(path)) failures.add(path);
240
+ }
241
+ if (validation.negation) failures.add('negated request');
242
+ if (failures.size) {
243
+ const detail = [...failures].sort().join(', ');
244
+ throw new ExtractionValidationError(
245
+ `extraction returned values not grounded in the input: ${detail}`);
246
+ }
247
+ return arguments_;
248
+ }
249
+
250
+ module.exports = {
251
+ ExtractionValidationError,
252
+ sourceYears,
253
+ relativeCue,
254
+ licensedYears,
255
+ walkGrounding,
256
+ temporalGrounding,
257
+ groundedNumberPaths,
258
+ sourceNumbers,
259
+ numericPaths,
260
+ ungroundedPaths,
261
+ annotateUngrounded,
262
+ validateExtraction,
263
+ };