@dxos/assistant 0.8.4-main.67995b8 → 0.8.4-main.a4bbb77
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/lib/browser/chunk-ILL2CN3G.mjs +374 -0
- package/dist/lib/browser/chunk-ILL2CN3G.mjs.map +7 -0
- package/dist/lib/browser/{chunk-6WRGDEMH.mjs → chunk-UB26N5UF.mjs} +2 -2
- package/dist/lib/browser/chunk-UB26N5UF.mjs.map +7 -0
- package/dist/lib/browser/extraction/index.mjs +32 -29
- package/dist/lib/browser/extraction/index.mjs.map +4 -4
- package/dist/lib/browser/index.mjs +682 -1035
- package/dist/lib/browser/index.mjs.map +4 -4
- package/dist/lib/browser/meta.json +1 -1
- package/dist/lib/browser/{named-entity-recognition-O46UTCLR.mjs → named-entity-recognition-QPYHML66.mjs} +2 -2
- package/dist/lib/node-esm/{chunk-3UU5V4FS.mjs → chunk-BCFXTNSK.mjs} +2 -2
- package/dist/lib/node-esm/chunk-BCFXTNSK.mjs.map +7 -0
- package/dist/lib/node-esm/chunk-GVKA7VL3.mjs +376 -0
- package/dist/lib/node-esm/chunk-GVKA7VL3.mjs.map +7 -0
- package/dist/lib/node-esm/extraction/index.mjs +32 -29
- package/dist/lib/node-esm/extraction/index.mjs.map +4 -4
- package/dist/lib/node-esm/index.mjs +682 -1035
- package/dist/lib/node-esm/index.mjs.map +4 -4
- package/dist/lib/node-esm/meta.json +1 -1
- package/dist/lib/node-esm/{named-entity-recognition-W4VMRCH6.mjs → named-entity-recognition-2HDF46YX.mjs} +2 -2
- package/dist/types/src/{context/binding.d.ts → conversation/context.d.ts} +45 -3
- package/dist/types/src/conversation/context.d.ts.map +1 -0
- package/dist/types/src/conversation/conversation.d.ts +24 -21
- package/dist/types/src/conversation/conversation.d.ts.map +1 -1
- package/dist/types/src/conversation/index.d.ts +1 -0
- package/dist/types/src/conversation/index.d.ts.map +1 -1
- package/dist/types/src/diff.d.ts +15 -0
- package/dist/types/src/diff.d.ts.map +1 -0
- package/dist/types/src/diff.test.d.ts +2 -0
- package/dist/types/src/diff.test.d.ts.map +1 -0
- package/dist/types/src/errors.d.ts +46 -9
- package/dist/types/src/errors.d.ts.map +1 -1
- package/dist/types/src/execution-graph.d.ts +82 -0
- package/dist/types/src/execution-graph.d.ts.map +1 -0
- package/dist/types/src/execution-graph.test.d.ts +2 -0
- package/dist/types/src/execution-graph.test.d.ts.map +1 -0
- package/dist/types/src/extraction/extraction-llm-function/extraction-llm-function.d.ts +1 -1
- package/dist/types/src/extraction/extraction-llm-function/extraction-llm-function.d.ts.map +1 -1
- package/dist/types/src/extraction/extraction-ner-function/extraction-ner-function.d.ts +94 -55
- package/dist/types/src/extraction/extraction-ner-function/extraction-ner-function.d.ts.map +1 -1
- package/dist/types/src/extraction/extraction-ner-function/named-entity-recognition.d.ts.map +1 -1
- package/dist/types/src/extraction/extraction.d.ts +250 -216
- package/dist/types/src/extraction/extraction.d.ts.map +1 -1
- package/dist/types/src/functions/services.d.ts +15 -3
- package/dist/types/src/functions/services.d.ts.map +1 -1
- package/dist/types/src/index.d.ts +3 -2
- package/dist/types/src/index.d.ts.map +1 -1
- package/dist/types/src/session/artifact-diff.d.ts +27 -0
- package/dist/types/src/session/artifact-diff.d.ts.map +1 -0
- package/dist/types/src/session/error-handling.d.ts +4 -0
- package/dist/types/src/session/error-handling.d.ts.map +1 -0
- package/dist/types/src/session/format.d.ts +184 -0
- package/dist/types/src/session/format.d.ts.map +1 -0
- package/dist/types/src/session/format.test.d.ts +2 -0
- package/dist/types/src/session/format.test.d.ts.map +1 -0
- package/dist/types/src/session/index.d.ts +3 -0
- package/dist/types/src/session/index.d.ts.map +1 -1
- package/dist/types/src/session/observer.d.ts +22 -0
- package/dist/types/src/session/observer.d.ts.map +1 -0
- package/dist/types/src/session/session.d.ts +23 -236
- package/dist/types/src/session/session.d.ts.map +1 -1
- package/dist/types/src/session/toolkit.d.ts +14 -0
- package/dist/types/src/session/toolkit.d.ts.map +1 -0
- package/dist/types/src/session/version-pin.d.ts +2 -2
- package/dist/types/src/session/version-pin.d.ts.map +1 -1
- package/dist/types/src/templates/index.d.ts +1 -1
- package/dist/types/src/templates/system.d.ts +18 -0
- package/dist/types/src/templates/system.d.ts.map +1 -0
- package/dist/types/src/types.d.ts +3 -0
- package/dist/types/src/types.d.ts.map +1 -1
- package/dist/types/tsconfig.tsbuildinfo +1 -1
- package/package.json +34 -33
- package/src/conversation/context.ts +154 -0
- package/src/conversation/conversation.ts +76 -53
- package/src/conversation/index.ts +1 -0
- package/src/diff.test.ts +43 -0
- package/src/diff.ts +79 -0
- package/src/errors.ts +6 -1
- package/src/execution-graph.test.ts +738 -0
- package/src/execution-graph.ts +500 -0
- package/src/extraction/extraction-llm-function/extraction-llm-function.test.ts +8 -7
- package/src/extraction/extraction-llm-function/extraction-llm-function.ts +12 -10
- package/src/extraction/extraction-ner-function/extraction-ner-function.test.ts +9 -9
- package/src/extraction/extraction-ner-function/extraction-ner-function.ts +13 -6
- package/src/extraction/extraction-ner-function/named-entity-recognition.test.ts +2 -1
- package/src/extraction/extraction-ner-function/named-entity-recognition.ts +2 -0
- package/src/extraction/quotes.test.ts +3 -3
- package/src/extraction/quotes.ts +2 -2
- package/src/functions/services.ts +97 -43
- package/src/index.ts +3 -2
- package/src/session/artifact-diff.ts +41 -0
- package/src/session/error-handling.ts +40 -0
- package/src/session/format.test.ts +60 -0
- package/src/session/format.ts +139 -0
- package/src/session/index.ts +3 -0
- package/src/session/observer.ts +58 -0
- package/src/session/session.conversations.json +1 -0
- package/src/session/session.test.ts +33 -197
- package/src/session/session.ts +119 -249
- package/src/session/toolkit.ts +46 -0
- package/src/session/version-pin.ts +4 -4
- package/src/summarization/summarization.test.ts +9 -7
- package/src/templates/format.tpl +60 -0
- package/src/templates/index.ts +1 -1
- package/src/templates/system.tpl +48 -0
- package/src/templates/system.ts +41 -0
- package/src/types.ts +10 -5
- package/src/typings.d.ts +5 -0
- package/dist/lib/browser/chunk-6WRGDEMH.mjs.map +0 -7
- package/dist/lib/browser/chunk-SIXTNSGV.mjs +0 -250
- package/dist/lib/browser/chunk-SIXTNSGV.mjs.map +0 -7
- package/dist/lib/node-esm/chunk-3UU5V4FS.mjs.map +0 -7
- package/dist/lib/node-esm/chunk-FD5NXKDP.mjs +0 -252
- package/dist/lib/node-esm/chunk-FD5NXKDP.mjs.map +0 -7
- package/dist/types/bin/chat.d.ts +0 -2
- package/dist/types/bin/chat.d.ts.map +0 -1
- package/dist/types/src/context/binder.d.ts +0 -34
- package/dist/types/src/context/binder.d.ts.map +0 -1
- package/dist/types/src/context/binding.d.ts.map +0 -1
- package/dist/types/src/context/index.d.ts +0 -3
- package/dist/types/src/context/index.d.ts.map +0 -1
- package/dist/types/src/research/exa.d.ts +0 -12
- package/dist/types/src/research/exa.d.ts.map +0 -1
- package/dist/types/src/research/graph.d.ts +0 -51
- package/dist/types/src/research/graph.d.ts.map +0 -1
- package/dist/types/src/research/graph.test.d.ts +0 -2
- package/dist/types/src/research/graph.test.d.ts.map +0 -1
- package/dist/types/src/research/index.d.ts +0 -4
- package/dist/types/src/research/index.d.ts.map +0 -1
- package/dist/types/src/research/research.d.ts +0 -10
- package/dist/types/src/research/research.d.ts.map +0 -1
- package/dist/types/src/research/research.test.d.ts +0 -2
- package/dist/types/src/research/research.test.d.ts.map +0 -1
- package/dist/types/src/session/version-pin.test.d.ts +0 -2
- package/dist/types/src/session/version-pin.test.d.ts.map +0 -1
- package/dist/types/src/templates/prompt.d.ts +0 -30
- package/dist/types/src/templates/prompt.d.ts.map +0 -1
- package/dist/types/src/testing/exa-search-1748337321991.d.ts +0 -38
- package/dist/types/src/testing/exa-search-1748337321991.d.ts.map +0 -1
- package/dist/types/src/testing/exa-search-1748337331526.d.ts +0 -37
- package/dist/types/src/testing/exa-search-1748337331526.d.ts.map +0 -1
- package/dist/types/src/testing/exa-search-1748337344119.d.ts +0 -58
- package/dist/types/src/testing/exa-search-1748337344119.d.ts.map +0 -1
- package/dist/types/src/testing/index.d.ts +0 -3
- package/dist/types/src/testing/index.d.ts.map +0 -1
- package/src/context/binder.ts +0 -103
- package/src/context/binding.ts +0 -32
- package/src/context/index.ts +0 -6
- package/src/research/exa.ts +0 -97
- package/src/research/graph.test.ts +0 -64
- package/src/research/graph.ts +0 -369
- package/src/research/index.ts +0 -7
- package/src/research/instructions-research.tpl +0 -98
- package/src/research/research.test.ts +0 -190
- package/src/research/research.ts +0 -87
- package/src/session/__snapshots__/session.test.ts.snap +0 -32
- package/src/session/version-pin.test.ts +0 -7
- package/src/templates/assistant.tpl +0 -111
- package/src/templates/prompt.ts +0 -45
- package/src/testing/exa-search-1748337321991.ts +0 -131
- package/src/testing/exa-search-1748337331526.ts +0 -144
- package/src/testing/exa-search-1748337344119.ts +0 -133
- package/src/testing/index.ts +0 -11
- package/src/typedefs.d.ts +0 -8
- /package/dist/lib/browser/{named-entity-recognition-O46UTCLR.mjs.map → named-entity-recognition-QPYHML66.mjs.map} +0 -0
- /package/dist/lib/node-esm/{named-entity-recognition-W4VMRCH6.mjs.map → named-entity-recognition-2HDF46YX.mjs.map} +0 -0
|
@@ -0,0 +1,374 @@
|
|
|
1
|
+
// src/session/artifact-diff.ts
|
|
2
|
+
import { Context } from "effect";
|
|
3
|
+
var ArtifactDiffResolver = class extends Context.Tag("@dxos/assistant/ArtifactDiffResolver")() {
|
|
4
|
+
};
|
|
5
|
+
|
|
6
|
+
// src/session/observer.ts
|
|
7
|
+
import { Effect, Function } from "effect";
|
|
8
|
+
var GenerationObserver = Object.freeze({
|
|
9
|
+
make: ({ onBegin = Function.constant(Effect.void), onPart = Function.constant(Effect.void), onBlock = Function.constant(Effect.void), onEnd = Function.constant(Effect.void), onMessage = Function.constant(Effect.void) } = {}) => ({
|
|
10
|
+
onBegin,
|
|
11
|
+
onPart,
|
|
12
|
+
onBlock,
|
|
13
|
+
onEnd,
|
|
14
|
+
onMessage
|
|
15
|
+
}),
|
|
16
|
+
noop: () => GenerationObserver.make(),
|
|
17
|
+
/**
|
|
18
|
+
* Debug printer to be used in unit-tests and browser devtools.
|
|
19
|
+
*/
|
|
20
|
+
fromPrinter: (printer) => GenerationObserver.make({
|
|
21
|
+
onBlock: (block) => Effect.sync(() => {
|
|
22
|
+
if (block.pending) {
|
|
23
|
+
return;
|
|
24
|
+
}
|
|
25
|
+
printer.printContentBlock(block);
|
|
26
|
+
}),
|
|
27
|
+
onMessage: (message) => Effect.sync(() => {
|
|
28
|
+
if (message.sender.role === "assistant") {
|
|
29
|
+
return;
|
|
30
|
+
}
|
|
31
|
+
printer.printMessage(message);
|
|
32
|
+
})
|
|
33
|
+
})
|
|
34
|
+
});
|
|
35
|
+
|
|
36
|
+
// src/errors.ts
|
|
37
|
+
import { BaseError } from "@dxos/errors";
|
|
38
|
+
var AiModelError = class extends BaseError.extend("AI_MODEL_ERROR", "AI model error") {
|
|
39
|
+
};
|
|
40
|
+
var AiAssistantError = class extends BaseError.extend("AI_ASSISTANT_ERROR", "AI assistant error") {
|
|
41
|
+
};
|
|
42
|
+
|
|
43
|
+
// src/session/session.ts
|
|
44
|
+
import { LanguageModel } from "@effect/ai";
|
|
45
|
+
import { Chunk, Effect as Effect3, Stream } from "effect";
|
|
46
|
+
import { AiParser, AiPreprocessor, callTool, getToolCalls } from "@dxos/ai";
|
|
47
|
+
import { todo } from "@dxos/debug";
|
|
48
|
+
import { Obj as Obj2 } from "@dxos/echo";
|
|
49
|
+
import { TracingService } from "@dxos/functions";
|
|
50
|
+
import { log as log2 } from "@dxos/log";
|
|
51
|
+
import { DataType as DataType2 } from "@dxos/schema";
|
|
52
|
+
|
|
53
|
+
// src/session/format.ts
|
|
54
|
+
import { Array, Effect as Effect2, Option, pipe } from "effect";
|
|
55
|
+
import { Template } from "@dxos/blueprints";
|
|
56
|
+
import { Obj } from "@dxos/echo";
|
|
57
|
+
import { ObjectVersion } from "@dxos/echo-db";
|
|
58
|
+
import { DatabaseService } from "@dxos/functions";
|
|
59
|
+
import { log } from "@dxos/log";
|
|
60
|
+
import { DataType } from "@dxos/schema";
|
|
61
|
+
import { trim } from "@dxos/util";
|
|
62
|
+
var __dxlog_file = "/__w/dxos/dxos/packages/core/assistant/src/session/format.ts";
|
|
63
|
+
var formatSystemPrompt = ({ system, blueprints = [], objects = [] }) => Effect2.gen(function* () {
|
|
64
|
+
const blueprintDefs = yield* pipe(blueprints, Effect2.forEach((blueprint) => Effect2.succeed(blueprint.instructions)), Effect2.flatMap(Effect2.forEach((template) => DatabaseService.loadOption(template.source))), Effect2.map(Array.filter(Option.isSome)), Effect2.map(Array.map((template) => trim`
|
|
65
|
+
<blueprint>
|
|
66
|
+
${Template.process(template.value.content)}
|
|
67
|
+
</blueprint>
|
|
68
|
+
`)), Effect2.map((blueprints2) => blueprints2.length > 0 ? [
|
|
69
|
+
"## Blueprints Definitions",
|
|
70
|
+
...blueprints2
|
|
71
|
+
].join("\n\n") : void 0));
|
|
72
|
+
const objectDefs = yield* pipe(objects, Effect2.forEach((object) => Effect2.succeed(trim`
|
|
73
|
+
<object>
|
|
74
|
+
<dxn>${Obj.getDXN(object)}</dxn>
|
|
75
|
+
<typename>${Obj.getTypename(object)}</typename>
|
|
76
|
+
</object>
|
|
77
|
+
`)), Effect2.map((objects2) => objects2.length > 0 ? [
|
|
78
|
+
"## Context Objects",
|
|
79
|
+
...objects2
|
|
80
|
+
].join("\n\n") : void 0));
|
|
81
|
+
return yield* pipe(Effect2.succeed([
|
|
82
|
+
system,
|
|
83
|
+
blueprintDefs,
|
|
84
|
+
objectDefs
|
|
85
|
+
].filter((def) => def !== void 0)), Effect2.map((parts) => parts.join("\n\n")));
|
|
86
|
+
});
|
|
87
|
+
var formatUserPrompt = ({ prompt, history = [] }) => Effect2.gen(function* () {
|
|
88
|
+
const blocks = [];
|
|
89
|
+
const artifactDiffResolver = yield* Effect2.serviceOption(ArtifactDiffResolver);
|
|
90
|
+
if (Option.isSome(artifactDiffResolver)) {
|
|
91
|
+
const versions = gatherObjectVersions(history);
|
|
92
|
+
const artifactDiff = yield* Effect2.tryPromise({
|
|
93
|
+
try: () => artifactDiffResolver.value.resolve([
|
|
94
|
+
...versions.entries()
|
|
95
|
+
].map(([id, version]) => ({
|
|
96
|
+
id,
|
|
97
|
+
lastVersion: version
|
|
98
|
+
}))),
|
|
99
|
+
catch: AiAssistantError.wrap({
|
|
100
|
+
message: "Artifact diff resolution error"
|
|
101
|
+
})
|
|
102
|
+
});
|
|
103
|
+
log("version", {
|
|
104
|
+
artifactDiff,
|
|
105
|
+
versions
|
|
106
|
+
}, {
|
|
107
|
+
F: __dxlog_file,
|
|
108
|
+
L: 90,
|
|
109
|
+
S: this,
|
|
110
|
+
C: (f, a) => f(...a)
|
|
111
|
+
});
|
|
112
|
+
for (const [id, { version }] of [
|
|
113
|
+
...artifactDiff.entries()
|
|
114
|
+
]) {
|
|
115
|
+
if (ObjectVersion.equals(version, versions.get(id))) {
|
|
116
|
+
artifactDiff.delete(id);
|
|
117
|
+
continue;
|
|
118
|
+
}
|
|
119
|
+
blocks.push({
|
|
120
|
+
_tag: "anchor",
|
|
121
|
+
objectId: id,
|
|
122
|
+
version
|
|
123
|
+
});
|
|
124
|
+
}
|
|
125
|
+
if (artifactDiff.size > 0) {
|
|
126
|
+
blocks.push(createArtifactUpdateBlock(artifactDiff));
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
return Obj.make(DataType.Message, {
|
|
130
|
+
created: (/* @__PURE__ */ new Date()).toISOString(),
|
|
131
|
+
sender: {
|
|
132
|
+
role: "user"
|
|
133
|
+
},
|
|
134
|
+
blocks: [
|
|
135
|
+
...blocks,
|
|
136
|
+
{
|
|
137
|
+
_tag: "text",
|
|
138
|
+
text: prompt
|
|
139
|
+
}
|
|
140
|
+
]
|
|
141
|
+
});
|
|
142
|
+
});
|
|
143
|
+
var gatherObjectVersions = (messages) => {
|
|
144
|
+
const artifactIds = /* @__PURE__ */ new Map();
|
|
145
|
+
for (const message of messages) {
|
|
146
|
+
for (const block of message.blocks) {
|
|
147
|
+
if (block._tag === "anchor") {
|
|
148
|
+
artifactIds.set(block.objectId, block.version);
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
return artifactIds;
|
|
153
|
+
};
|
|
154
|
+
var createArtifactUpdateBlock = (artifactDiff) => {
|
|
155
|
+
return {
|
|
156
|
+
_tag: "text",
|
|
157
|
+
// TODO(dmaretskyi): Does this need to be a special content-block?
|
|
158
|
+
disposition: "artifact-update",
|
|
159
|
+
text: trim`
|
|
160
|
+
The following artifacts have been updated since the last message:
|
|
161
|
+
${[
|
|
162
|
+
...artifactDiff.entries()
|
|
163
|
+
].map(([id, { diff }]) => `<changed-artifact id="${id}">${diff ? `
|
|
164
|
+
${diff}` : ""}</changed-artifact>`).join("\n")}
|
|
165
|
+
`
|
|
166
|
+
};
|
|
167
|
+
};
|
|
168
|
+
|
|
169
|
+
// src/session/session.ts
|
|
170
|
+
function _define_property(obj, key, value) {
|
|
171
|
+
if (key in obj) {
|
|
172
|
+
Object.defineProperty(obj, key, {
|
|
173
|
+
value,
|
|
174
|
+
enumerable: true,
|
|
175
|
+
configurable: true,
|
|
176
|
+
writable: true
|
|
177
|
+
});
|
|
178
|
+
} else {
|
|
179
|
+
obj[key] = value;
|
|
180
|
+
}
|
|
181
|
+
return obj;
|
|
182
|
+
}
|
|
183
|
+
var __dxlog_file2 = "/__w/dxos/dxos/packages/core/assistant/src/session/session.ts";
|
|
184
|
+
var AiSession = class {
|
|
185
|
+
/**
|
|
186
|
+
* @deprecated
|
|
187
|
+
*/
|
|
188
|
+
// TODO(burdon): Implement or remove.
|
|
189
|
+
async runStructured(_schema, _options) {
|
|
190
|
+
return todo();
|
|
191
|
+
}
|
|
192
|
+
constructor(_options = {}) {
|
|
193
|
+
_define_property(this, "_options", void 0);
|
|
194
|
+
_define_property(this, "_semaphore", void 0);
|
|
195
|
+
_define_property(this, "_history", void 0);
|
|
196
|
+
_define_property(this, "_pending", void 0);
|
|
197
|
+
_define_property(this, "run", void 0);
|
|
198
|
+
this._options = _options;
|
|
199
|
+
this._semaphore = Effect3.runSync(Effect3.makeSemaphore(1));
|
|
200
|
+
this._history = [];
|
|
201
|
+
this._pending = [];
|
|
202
|
+
this.run = ({ prompt, system: systemTemplate, history = [], objects = [], blueprints = [], toolkit, observer = GenerationObserver.noop() }) => Effect3.gen(this, function* () {
|
|
203
|
+
this._history = [
|
|
204
|
+
...history
|
|
205
|
+
];
|
|
206
|
+
this._pending = [];
|
|
207
|
+
const pending = this._pending;
|
|
208
|
+
const submitMessage = Effect3.fnUntraced(function* (message) {
|
|
209
|
+
pending.push(message);
|
|
210
|
+
yield* observer.onMessage(message);
|
|
211
|
+
yield* TracingService.emitConverationMessage(message);
|
|
212
|
+
return message;
|
|
213
|
+
});
|
|
214
|
+
const promptMessage = yield* submitMessage(yield* formatUserPrompt({
|
|
215
|
+
prompt,
|
|
216
|
+
history
|
|
217
|
+
}));
|
|
218
|
+
const system = yield* formatSystemPrompt({
|
|
219
|
+
system: systemTemplate,
|
|
220
|
+
blueprints,
|
|
221
|
+
objects
|
|
222
|
+
});
|
|
223
|
+
do {
|
|
224
|
+
log2("request", {
|
|
225
|
+
prompt: promptMessage,
|
|
226
|
+
system: {
|
|
227
|
+
snippet: createSnippet(system),
|
|
228
|
+
length: system.length
|
|
229
|
+
},
|
|
230
|
+
pending: this._pending.length,
|
|
231
|
+
history: this._history.length,
|
|
232
|
+
objects: objects?.length ?? 0
|
|
233
|
+
}, {
|
|
234
|
+
F: __dxlog_file2,
|
|
235
|
+
L: 103,
|
|
236
|
+
S: this,
|
|
237
|
+
C: (f, a) => f(...a)
|
|
238
|
+
});
|
|
239
|
+
const prompt2 = yield* AiPreprocessor.preprocessPrompt([
|
|
240
|
+
...this._history,
|
|
241
|
+
...this._pending
|
|
242
|
+
], {
|
|
243
|
+
system
|
|
244
|
+
});
|
|
245
|
+
const blocks = yield* LanguageModel.streamText({
|
|
246
|
+
prompt: prompt2,
|
|
247
|
+
toolkit,
|
|
248
|
+
disableToolCallResolution: true
|
|
249
|
+
}).pipe(
|
|
250
|
+
// TOOD(dmaretskyi): Error mapping.
|
|
251
|
+
AiParser.parseResponse({
|
|
252
|
+
onBegin: () => observer.onBegin(),
|
|
253
|
+
onBlock: (block) => observer.onBlock(block),
|
|
254
|
+
onPart: (part) => observer.onPart(part),
|
|
255
|
+
onEnd: (summary) => observer.onEnd(summary)
|
|
256
|
+
}),
|
|
257
|
+
Stream.runCollect,
|
|
258
|
+
Effect3.map(Chunk.toArray)
|
|
259
|
+
);
|
|
260
|
+
const response = yield* submitMessage(Obj2.make(DataType2.Message, {
|
|
261
|
+
created: (/* @__PURE__ */ new Date()).toISOString(),
|
|
262
|
+
sender: {
|
|
263
|
+
role: "assistant"
|
|
264
|
+
},
|
|
265
|
+
blocks
|
|
266
|
+
}));
|
|
267
|
+
const toolCalls = getToolCalls(response);
|
|
268
|
+
if (toolCalls.length === 0) {
|
|
269
|
+
break;
|
|
270
|
+
} else if (!toolkit) {
|
|
271
|
+
throw new Error("No toolkit provided");
|
|
272
|
+
}
|
|
273
|
+
const toolResults = yield* Effect3.forEach(toolCalls, (toolCall) => {
|
|
274
|
+
return callTool(toolkit, toolCall).pipe(Effect3.provide(TracingService.layerSubframe((context) => ({
|
|
275
|
+
...context,
|
|
276
|
+
parentMessage: response.id,
|
|
277
|
+
toolCallId: toolCall.toolCallId
|
|
278
|
+
}))));
|
|
279
|
+
});
|
|
280
|
+
yield* submitMessage(Obj2.make(DataType2.Message, {
|
|
281
|
+
created: (/* @__PURE__ */ new Date()).toISOString(),
|
|
282
|
+
sender: {
|
|
283
|
+
role: "tool"
|
|
284
|
+
},
|
|
285
|
+
blocks: toolResults
|
|
286
|
+
}));
|
|
287
|
+
} while (true);
|
|
288
|
+
log2("done", {
|
|
289
|
+
pending: this._pending.length
|
|
290
|
+
}, {
|
|
291
|
+
F: __dxlog_file2,
|
|
292
|
+
L: 174,
|
|
293
|
+
S: this,
|
|
294
|
+
C: (f, a) => f(...a)
|
|
295
|
+
});
|
|
296
|
+
return this._pending;
|
|
297
|
+
}).pipe(this._semaphore.withPermits(1), Effect3.withSpan("AiSession.run"));
|
|
298
|
+
}
|
|
299
|
+
};
|
|
300
|
+
var createSnippet = (text, len = 32) => text.length <= len * 2 ? text : [
|
|
301
|
+
text.slice(0, len),
|
|
302
|
+
"...",
|
|
303
|
+
text.slice(-len)
|
|
304
|
+
].join("");
|
|
305
|
+
|
|
306
|
+
// src/session/toolkit.ts
|
|
307
|
+
import { Toolkit } from "@effect/ai";
|
|
308
|
+
import { Effect as Effect4 } from "effect";
|
|
309
|
+
import { ToolExecutionService, ToolResolverService } from "@dxos/ai";
|
|
310
|
+
import { isTruthy } from "@dxos/util";
|
|
311
|
+
var createToolkit = ({ toolkit: toolkitParam, toolIds = [], blueprints = [] }) => Effect4.gen(function* () {
|
|
312
|
+
const blueprintToolkit = yield* ToolResolverService.resolveToolkit([
|
|
313
|
+
...blueprints.flatMap(({ tools }) => tools),
|
|
314
|
+
...toolIds
|
|
315
|
+
]);
|
|
316
|
+
const blueprintToolkitHandler = yield* blueprintToolkit.toContext(ToolExecutionService.handlersFor(blueprintToolkit));
|
|
317
|
+
const toolkit = Toolkit.merge(...[
|
|
318
|
+
toolkitParam,
|
|
319
|
+
blueprintToolkit
|
|
320
|
+
].filter(isTruthy));
|
|
321
|
+
return yield* toolkit.pipe(Effect4.provide(blueprintToolkitHandler));
|
|
322
|
+
});
|
|
323
|
+
|
|
324
|
+
// src/session/version-pin.ts
|
|
325
|
+
import { Schema } from "effect";
|
|
326
|
+
import { getVersion } from "@dxos/echo-db";
|
|
327
|
+
import { ObjectId } from "@dxos/echo-schema";
|
|
328
|
+
function _define_property2(obj, key, value) {
|
|
329
|
+
if (key in obj) {
|
|
330
|
+
Object.defineProperty(obj, key, {
|
|
331
|
+
value,
|
|
332
|
+
enumerable: true,
|
|
333
|
+
configurable: true,
|
|
334
|
+
writable: true
|
|
335
|
+
});
|
|
336
|
+
} else {
|
|
337
|
+
obj[key] = value;
|
|
338
|
+
}
|
|
339
|
+
return obj;
|
|
340
|
+
}
|
|
341
|
+
var _class;
|
|
342
|
+
var ObjectVersionSchema = Schema.Unknown;
|
|
343
|
+
var VersionPinSchema = Schema.Struct({
|
|
344
|
+
// TODO(dmaretskyi): Use Ref when those support encoding.
|
|
345
|
+
objectId: ObjectId,
|
|
346
|
+
// TODO(dmaretskyi): Could be opaque.
|
|
347
|
+
version: ObjectVersionSchema
|
|
348
|
+
});
|
|
349
|
+
var VersionPin = (_class = class extends VersionPinSchema {
|
|
350
|
+
static fromObject(object) {
|
|
351
|
+
return VersionPin.make({
|
|
352
|
+
objectId: object.id,
|
|
353
|
+
version: getVersion(object)
|
|
354
|
+
});
|
|
355
|
+
}
|
|
356
|
+
static createBlock(pin) {
|
|
357
|
+
return {
|
|
358
|
+
_tag: "json",
|
|
359
|
+
disposition: VersionPin.DISPOSITION,
|
|
360
|
+
data: JSON.stringify(pin)
|
|
361
|
+
};
|
|
362
|
+
}
|
|
363
|
+
}, _define_property2(_class, "DISPOSITION", "version-pin"), _class);
|
|
364
|
+
|
|
365
|
+
export {
|
|
366
|
+
ArtifactDiffResolver,
|
|
367
|
+
GenerationObserver,
|
|
368
|
+
AiModelError,
|
|
369
|
+
AiAssistantError,
|
|
370
|
+
AiSession,
|
|
371
|
+
createToolkit,
|
|
372
|
+
VersionPin
|
|
373
|
+
};
|
|
374
|
+
//# sourceMappingURL=chunk-ILL2CN3G.mjs.map
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
{
|
|
2
|
+
"version": 3,
|
|
3
|
+
"sources": ["../../../src/session/artifact-diff.ts", "../../../src/session/observer.ts", "../../../src/errors.ts", "../../../src/session/session.ts", "../../../src/session/format.ts", "../../../src/session/toolkit.ts", "../../../src/session/version-pin.ts"],
|
|
4
|
+
"sourcesContent": ["//\n// Copyright 2025 DXOS.org\n//\n\nimport { Context } from 'effect';\n\nimport { type ObjectVersion } from '@dxos/echo-db';\nimport { type ObjectId } from '@dxos/echo-schema';\n\n/**\n * Resolves artifact ids to their versions.\n * Used to give the model a sense of the changes to the artifacts made by users during the conversation.\n * The artifacts versions are pinned in the history, and whenever the artifact changes in-between assistant's steps,\n * a diff is inserted into the conversation.\n *\n * Can be optionally provided to the session run call.\n */\n// TODO(dmaretskyi): Convert to Context.Reference\nexport class ArtifactDiffResolver extends Context.Tag('@dxos/assistant/ArtifactDiffResolver')<\n ArtifactDiffResolver,\n ArtifactDiffResolver.Service\n>() {}\n\nexport namespace ArtifactDiffResolver {\n export type Service = {\n resolve: (\n artifacts: {\n id: ObjectId;\n lastVersion: ObjectVersion;\n }[],\n ) => Promise<\n Map<\n ObjectId,\n {\n version: ObjectVersion;\n diff?: string;\n }\n >\n >;\n };\n}\n", "//\n// Copyright 2025 DXOS.org\n//\n\nimport { Effect, Function } from 'effect';\n\nimport { type AiParser, type ConsolePrinter } from '@dxos/ai';\nimport { type DataType } from '@dxos/schema';\n\n/**\n * Live observer of the generation process.\n */\nexport interface GenerationObserver extends AiParser.ParseResponseCallbacks {\n /**\n * Complete messages fired during the session, both from the model and from the user.\n * This message will contain all message blocks emitted through the `onBlock` callback.\n */\n onMessage: (message: DataType.Message) => Effect.Effect<void>;\n}\n\nexport const GenerationObserver = Object.freeze({\n make: ({\n onBegin = Function.constant(Effect.void),\n onPart = Function.constant(Effect.void),\n onBlock = Function.constant(Effect.void),\n onEnd = Function.constant(Effect.void),\n onMessage = Function.constant(Effect.void),\n }: Partial<GenerationObserver> = {}): GenerationObserver => ({\n onBegin,\n onPart,\n onBlock,\n onEnd,\n onMessage,\n }),\n\n noop: () => GenerationObserver.make(),\n\n /**\n * Debug printer to be used in unit-tests and browser devtools.\n */\n fromPrinter: (printer: ConsolePrinter) =>\n GenerationObserver.make({\n onBlock: (block) =>\n Effect.sync(() => {\n if (block.pending) {\n return; // Only prints full blocks (better for unit-tests and browser devtools).\n }\n printer.printContentBlock(block);\n }),\n onMessage: (message) =>\n Effect.sync(() => {\n if (message.sender.role === 'assistant') {\n return; // Skip assistant messages since they are printed in the `onBlock` callback.\n }\n printer.printMessage(message);\n }),\n }),\n});\n", "//\n// Copyright 2025 DXOS.org\n//\n\nimport { BaseError } from '@dxos/errors';\n\n/**\n * Generic error from AI model.\n */\nexport class AiModelError extends BaseError.extend('AI_MODEL_ERROR', 'AI model error') {}\n\n/**\n * Generic error for AI agent execution.\n */\nexport class AiAssistantError extends BaseError.extend('AI_ASSISTANT_ERROR', 'AI assistant error') {}\n", "//\n// Copyright 2025 DXOS.org\n//\n\nimport { type AiError, LanguageModel, type Tool, type Toolkit } from '@effect/ai';\nimport { Chunk, Effect, type Schema, Stream } from 'effect';\n\nimport {\n AiParser,\n AiPreprocessor,\n type AiToolNotFoundError,\n type PromptPreprocessingError,\n type ToolExecutionService,\n type ToolResolverService,\n callTool,\n getToolCalls,\n} from '@dxos/ai';\nimport { type Blueprint } from '@dxos/blueprints';\nimport { todo } from '@dxos/debug';\nimport { Obj } from '@dxos/echo';\nimport { TracingService } from '@dxos/functions';\nimport { log } from '@dxos/log';\nimport { DataType } from '@dxos/schema';\n\nimport { type AiAssistantError } from '../errors';\n\nimport { formatSystemPrompt, formatUserPrompt } from './format';\nimport { GenerationObserver } from './observer';\n\nexport type AiSessionRunError = AiError.AiError | PromptPreprocessingError | AiToolNotFoundError | AiAssistantError;\n\nexport type AiSessionRunRequirements =\n | LanguageModel.LanguageModel\n | ToolExecutionService\n | ToolResolverService\n | TracingService;\n\nexport type AiSessionRunParams<Tools extends Record<string, Tool.Any>> = {\n prompt: string;\n system?: string;\n history?: DataType.Message[];\n objects?: Obj.Any[];\n blueprints?: Blueprint.Blueprint[];\n toolkit?: Toolkit.WithHandler<Tools>;\n observer?: GenerationObserver;\n};\n\nexport type AiSessionOptions = {};\n\n/**\n * Contains message history, tools, current context.\n * Current context means the state of the app, time of day, and other contextual information.\n * It makes requests to the model, its a state machine.\n * It keeps track of the current goal.\n * It manages the context window.\n * Tracks the success criteria of reaching the goal, exposing metrics (stretch).\n * Could be run locally in the app or remotely.\n * Could be personal or shared.\n */\nexport class AiSession {\n /** Prevents concurrent execution of session. */\n private readonly _semaphore = Effect.runSync(Effect.makeSemaphore(1));\n\n /** Message history from queue. */\n // TODO(burdon): Evolve into supporting a git-like graph of messages.\n private _history: DataType.Message[] = [];\n\n /** Pending messages for this session (incl. the current prompt). */\n private _pending: DataType.Message[] = [];\n\n constructor(private readonly _options: AiSessionOptions = {}) {}\n\n run = <Tools extends Record<string, Tool.Any>>({\n prompt,\n system: systemTemplate,\n history = [],\n objects = [],\n blueprints = [],\n toolkit,\n observer = GenerationObserver.noop(),\n }: AiSessionRunParams<Tools>): Effect.Effect<DataType.Message[], AiSessionRunError, AiSessionRunRequirements> =>\n Effect.gen(this, function* () {\n this._history = [...history];\n this._pending = [];\n const pending = this._pending;\n\n const submitMessage = Effect.fnUntraced(function* (message: DataType.Message) {\n pending.push(message);\n yield* observer.onMessage(message);\n yield* TracingService.emitConverationMessage(message);\n return message;\n });\n\n // Submit the prompt.\n // TODO(burdon): Remove if cancelled?\n const promptMessage = yield* submitMessage(yield* formatUserPrompt({ prompt, history }));\n\n // Generate system and prompt messages.\n const system = yield* formatSystemPrompt({ system: systemTemplate, blueprints, objects });\n\n // Tool call loop.\n do {\n log('request', {\n prompt: promptMessage,\n system: { snippet: createSnippet(system), length: system.length },\n pending: this._pending.length,\n history: this._history.length,\n objects: objects?.length ?? 0,\n });\n\n const prompt = yield* AiPreprocessor.preprocessPrompt([...this._history, ...this._pending], { system });\n\n // Execute the stream request.\n // logDump('prompt', Prompt.Prompt.pipe(Schema.encodeSync)(prompt));\n const blocks = yield* LanguageModel.streamText({\n prompt,\n toolkit,\n disableToolCallResolution: true,\n }).pipe(\n // TOOD(dmaretskyi): Error mapping.\n AiParser.parseResponse({\n onBegin: () => observer.onBegin(),\n onBlock: (block) => observer.onBlock(block),\n onPart: (part) => observer.onPart(part),\n onEnd: (summary) => observer.onEnd(summary),\n }),\n Stream.runCollect,\n Effect.map(Chunk.toArray),\n );\n\n // Create the response message.\n const response = yield* submitMessage(\n Obj.make(DataType.Message, {\n created: new Date().toISOString(),\n sender: { role: 'assistant' },\n blocks,\n }),\n );\n\n // Parse the response for tool calls.\n const toolCalls = getToolCalls(response);\n if (toolCalls.length === 0) {\n break;\n } else if (!toolkit) {\n throw new Error('No toolkit provided'); // TODO(burdon): Throw user error?\n }\n\n // Execute the tool calls.\n // TODO(burdon): Retry errors? Write result when each completes individually?\n const toolResults = yield* Effect.forEach(toolCalls, (toolCall) => {\n return callTool(toolkit, toolCall).pipe(\n Effect.provide(\n TracingService.layerSubframe((context) => ({\n ...context,\n parentMessage: response.id,\n toolCallId: toolCall.toolCallId,\n })),\n ),\n );\n });\n\n // Add to queue and continue loop.\n // TODO(wittjosiah): Sometimes tool error results are added to the queue before the tool agent statuses.\n // This results in a broken execution graph.\n yield* submitMessage(\n Obj.make(DataType.Message, {\n created: new Date().toISOString(),\n sender: { role: 'tool' },\n blocks: toolResults,\n }),\n );\n } while (true);\n\n log('done', { pending: this._pending.length });\n return this._pending;\n }).pipe(this._semaphore.withPermits(1), Effect.withSpan('AiSession.run'));\n\n /**\n * @deprecated\n */\n // TODO(burdon): Implement or remove.\n async runStructured<S extends Schema.Schema.AnyNoContext>(\n _schema: S,\n _options: AiSessionRunParams<any>,\n ): Promise<Schema.Schema.Type<S>> {\n return todo();\n // const parser = structuredOutputParser(schema);\n // const result = await this.run({\n // ...options,\n // executableTools: [...(options.executableTools ?? []), parser.tool],\n // });\n // return parser.getResult(result);\n }\n}\n\nconst createSnippet = (text: string, len = 32) =>\n text.length <= len * 2 ? text : [text.slice(0, len), '...', text.slice(-len)].join('');\n\n// TODO(dmaretskyi): Extract as a general util.\nconst logDump = (message: string, data: unknown) => {\n // eslint-disable-next-line @typescript-eslint/no-require-imports\n const { writeFileSync } = require('node:fs');\n const path = `/tmp/log-data-${Date.now()}.json`;\n writeFileSync(path, JSON.stringify(data, null, 2));\n log.info(message, { path });\n};\n", "//\n// Copyright 2025 DXOS.org\n//\n\nimport { Array, Effect, Option, pipe } from 'effect';\n\nimport { Template } from '@dxos/blueprints';\nimport { Obj } from '@dxos/echo';\nimport { ObjectVersion } from '@dxos/echo-db';\nimport { type ObjectId } from '@dxos/echo-schema';\nimport { DatabaseService } from '@dxos/functions';\nimport { log } from '@dxos/log';\nimport { type ContentBlock, DataType } from '@dxos/schema';\nimport { trim } from '@dxos/util';\n\nimport { AiAssistantError } from '../errors';\n\nimport { ArtifactDiffResolver } from './artifact-diff';\nimport { type AiSessionRunParams } from './session';\n\n/**\n * Formats the system prompt.\n */\n// TODO(burdon): Move to AiPreprocessor.\nexport const formatSystemPrompt = ({\n system,\n blueprints = [],\n objects = [],\n}: Pick<AiSessionRunParams<any>, 'system' | 'blueprints' | 'objects'>) =>\n Effect.gen(function* () {\n const blueprintDefs = yield* pipe(\n blueprints,\n Effect.forEach((blueprint) => Effect.succeed(blueprint.instructions)),\n Effect.flatMap(Effect.forEach((template) => DatabaseService.loadOption(template.source))),\n Effect.map(Array.filter(Option.isSome)),\n Effect.map(\n Array.map(\n (template) => trim`\n <blueprint>\n ${Template.process(template.value.content)}\n </blueprint>\n `,\n ),\n ),\n Effect.map((blueprints) =>\n blueprints.length > 0 ? ['## Blueprints Definitions', ...blueprints].join('\\n\\n') : undefined,\n ),\n );\n\n const objectDefs = yield* pipe(\n objects,\n Effect.forEach((object) =>\n Effect.succeed(trim`\n <object>\n <dxn>${Obj.getDXN(object)}</dxn>\n <typename>${Obj.getTypename(object)}</typename>\n </object>\n `),\n ),\n Effect.map((objects) => (objects.length > 0 ? ['## Context Objects', ...objects].join('\\n\\n') : undefined)),\n );\n\n return yield* pipe(\n Effect.succeed([system, blueprintDefs, objectDefs].filter((def): def is string => def !== undefined)),\n Effect.map((parts) => parts.join('\\n\\n')),\n );\n });\n\n/**\n * Formats the user prompt.\n */\n// TODO(burdon): Move to AiPreprocessor.\n// TODO(burdon): Convert util below to `Effect.fn` (to preserve stack info)\nexport const formatUserPrompt = ({ prompt, history = [] }: Pick<AiSessionRunParams<any>, 'prompt' | 'history'>) =>\n Effect.gen(function* () {\n const blocks: ContentBlock.Any[] = [];\n\n // TODO(dmaretskyi): Evaluate other approaches as `serviceOption` isn't represented in the type system.\n const artifactDiffResolver = yield* Effect.serviceOption(ArtifactDiffResolver);\n if (Option.isSome(artifactDiffResolver)) {\n const versions = gatherObjectVersions(history);\n const artifactDiff = yield* Effect.tryPromise({\n try: () =>\n artifactDiffResolver.value.resolve(\n [...versions.entries()].map(([id, version]) => ({ id, lastVersion: version })),\n ),\n catch: AiAssistantError.wrap({ message: 'Artifact diff resolution error' }),\n });\n\n log('version', { artifactDiff, versions });\n for (const [id, { version }] of [...artifactDiff.entries()]) {\n if (ObjectVersion.equals(version, versions.get(id)!)) {\n artifactDiff.delete(id);\n continue;\n }\n\n blocks.push({ _tag: 'anchor', objectId: id, version });\n }\n\n if (artifactDiff.size > 0) {\n blocks.push(createArtifactUpdateBlock(artifactDiff));\n }\n }\n\n return Obj.make(DataType.Message, {\n created: new Date().toISOString(),\n sender: { role: 'user' },\n blocks: [...blocks, { _tag: 'text', text: prompt }],\n });\n });\n\nconst gatherObjectVersions = (messages: DataType.Message[]): Map<ObjectId, ObjectVersion> => {\n const artifactIds = new Map<ObjectId, ObjectVersion>();\n for (const message of messages) {\n for (const block of message.blocks) {\n if (block._tag === 'anchor') {\n artifactIds.set(block.objectId, block.version as ObjectVersion);\n }\n }\n }\n\n return artifactIds;\n};\n\nconst createArtifactUpdateBlock = (\n artifactDiff: Map<ObjectId, { version: ObjectVersion; diff?: string }>,\n): ContentBlock.Any => {\n return {\n _tag: 'text',\n // TODO(dmaretskyi): Does this need to be a special content-block?\n disposition: 'artifact-update',\n text: trim`\n The following artifacts have been updated since the last message:\n ${[...artifactDiff.entries()]\n .map(([id, { diff }]) => `<changed-artifact id=\"${id}\">${diff ? `\\n${diff}` : ''}</changed-artifact>`)\n .join('\\n')}\n `,\n };\n};\n", "//\n// Copyright 2025 DXOS.org\n//\n\nimport { type Tool, Toolkit } from '@effect/ai';\nimport { type Context, Effect } from 'effect';\n\nimport { type AiToolNotFoundError, ToolExecutionService, type ToolId, ToolResolverService } from '@dxos/ai';\nimport { type Blueprint } from '@dxos/blueprints';\nimport { isTruthy } from '@dxos/util';\n\nexport type ToolkitParams<Tools extends Record<string, Tool.Any>> = {\n toolkit?: Toolkit.Toolkit<Tools>;\n toolIds?: ToolId[];\n blueprints?: Blueprint.Blueprint[];\n};\n\n/**\n * Build a combined toolkit from the blueprint tools and the provided toolkit.\n */\nexport const createToolkit = <Tools extends Record<string, Tool.Any> = {}>({\n toolkit: toolkitParam,\n toolIds = [],\n blueprints = [],\n}: ToolkitParams<Tools>): Effect.Effect<\n Toolkit.WithHandler<any>,\n AiToolNotFoundError,\n ToolResolverService | ToolExecutionService | Tool.HandlersFor<Tools>\n> =>\n Effect.gen(function* () {\n const blueprintToolkit = yield* ToolResolverService.resolveToolkit([\n ...blueprints.flatMap(({ tools }) => tools),\n ...toolIds,\n ]);\n\n const blueprintToolkitHandler: Context.Context<Tool.HandlersFor<any>> = yield* blueprintToolkit.toContext(\n ToolExecutionService.handlersFor(blueprintToolkit),\n );\n\n const toolkit = Toolkit.merge(...[toolkitParam, blueprintToolkit].filter(isTruthy));\n return yield* toolkit.pipe(Effect.provide(blueprintToolkitHandler)) as any as Effect.Effect<\n Toolkit.WithHandler<any>,\n never,\n Tool.HandlersFor<Tools>\n >;\n });\n", "//\n// Copyright 2025 DXOS.org\n//\n\nimport { Schema } from 'effect';\n\nimport { type ObjectVersion } from '@dxos/echo-db';\nimport { getVersion } from '@dxos/echo-db';\nimport { type AnyEchoObject, ObjectId } from '@dxos/echo-schema';\nimport { type ContentBlock } from '@dxos/schema';\n\n// TODO(dmaretskyi): Extract.\nconst ObjectVersionSchema = Schema.Unknown as Schema.Schema<ObjectVersion>;\n\nconst VersionPinSchema = Schema.Struct({\n // TODO(dmaretskyi): Use Ref when those support encoding.\n objectId: ObjectId,\n // TODO(dmaretskyi): Could be opaque.\n version: ObjectVersionSchema,\n});\n\n/**\n * Used to pin a specific version of an object during the course of a conversation.\n * This allows us to detect when the object changes during the course of the conversation.\n */\nexport interface VersionPin extends Schema.Schema.Type<typeof VersionPinSchema> {}\n\nexport const VersionPin: typeof VersionPinSchema & {\n DISPOSITION: 'version-pin';\n fromObject: (object: AnyEchoObject) => VersionPin;\n createBlock: (pin: VersionPin) => ContentBlock.Any;\n} = class extends VersionPinSchema {\n static readonly DISPOSITION = 'version-pin';\n static fromObject(object: AnyEchoObject): VersionPin {\n return VersionPin.make({\n objectId: object.id,\n version: getVersion(object),\n });\n }\n\n static createBlock(pin: VersionPin): ContentBlock.Any {\n return {\n _tag: 'json',\n disposition: VersionPin.DISPOSITION,\n data: JSON.stringify(pin),\n };\n }\n};\n"],
|
|
5
|
+
"mappings": ";AAIA,SAASA,eAAe;AAcjB,IAAMC,uBAAN,cAAmCC,QAAQC,IAAI,sCAAA,EAAA,EAAA;AAGjD;;;ACjBL,SAASC,QAAQC,gBAAgB;AAgB1B,IAAMC,qBAAqBC,OAAOC,OAAO;EAC9CC,MAAM,CAAC,EACLC,UAAUC,SAASC,SAASC,OAAOC,IAAI,GACvCC,SAASJ,SAASC,SAASC,OAAOC,IAAI,GACtCE,UAAUL,SAASC,SAASC,OAAOC,IAAI,GACvCG,QAAQN,SAASC,SAASC,OAAOC,IAAI,GACrCI,YAAYP,SAASC,SAASC,OAAOC,IAAI,EAAC,IACX,CAAC,OAA2B;IAC3DJ;IACAK;IACAC;IACAC;IACAC;EACF;EAEAC,MAAM,MAAMb,mBAAmBG,KAAI;;;;EAKnCW,aAAa,CAACC,YACZf,mBAAmBG,KAAK;IACtBO,SAAS,CAACM,UACRT,OAAOU,KAAK,MAAA;AACV,UAAID,MAAME,SAAS;AACjB;MACF;AACAH,cAAQI,kBAAkBH,KAAAA;IAC5B,CAAA;IACFJ,WAAW,CAACQ,YACVb,OAAOU,KAAK,MAAA;AACV,UAAIG,QAAQC,OAAOC,SAAS,aAAa;AACvC;MACF;AACAP,cAAQQ,aAAaH,OAAAA;IACvB,CAAA;EACJ,CAAA;AACJ,CAAA;;;ACrDA,SAASI,iBAAiB;AAKnB,IAAMC,eAAN,cAA2BC,UAAUC,OAAO,kBAAkB,gBAAA,EAAA;AAAmB;AAKjF,IAAMC,mBAAN,cAA+BF,UAAUC,OAAO,sBAAsB,oBAAA,EAAA;AAAuB;;;ACVpG,SAAuBE,qBAA8C;AACrE,SAASC,OAAOC,UAAAA,SAAqBC,cAAc;AAEnD,SACEC,UACAC,gBAKAC,UACAC,oBACK;AAEP,SAASC,YAAY;AACrB,SAASC,OAAAA,YAAW;AACpB,SAASC,sBAAsB;AAC/B,SAASC,OAAAA,YAAW;AACpB,SAASC,YAAAA,iBAAgB;;;AClBzB,SAASC,OAAOC,UAAAA,SAAQC,QAAQC,YAAY;AAE5C,SAASC,gBAAgB;AACzB,SAASC,WAAW;AACpB,SAASC,qBAAqB;AAE9B,SAASC,uBAAuB;AAChC,SAASC,WAAW;AACpB,SAA4BC,gBAAgB;AAC5C,SAASC,YAAY;;AAWd,IAAMC,qBAAqB,CAAC,EACjCC,QACAC,aAAa,CAAA,GACbC,UAAU,CAAA,EAAE,MAEZC,QAAOC,IAAI,aAAA;AACT,QAAMC,gBAAgB,OAAOC,KAC3BL,YACAE,QAAOI,QAAQ,CAACC,cAAcL,QAAOM,QAAQD,UAAUE,YAAY,CAAA,GACnEP,QAAOQ,QAAQR,QAAOI,QAAQ,CAACK,aAAaC,gBAAgBC,WAAWF,SAASG,MAAM,CAAA,CAAA,GACtFZ,QAAOa,IAAIC,MAAMC,OAAOC,OAAOC,MAAM,CAAA,GACrCjB,QAAOa,IACLC,MAAMD,IACJ,CAACJ,aAAaS;;gBAERC,SAASC,QAAQX,SAASY,MAAMC,OAAO,CAAA;;WAE5C,CAAA,GAGLtB,QAAOa,IAAI,CAACf,gBACVA,YAAWyB,SAAS,IAAI;IAAC;OAAgCzB;IAAY0B,KAAK,MAAA,IAAUC,MAAAA,CAAAA;AAIxF,QAAMC,aAAa,OAAOvB,KACxBJ,SACAC,QAAOI,QAAQ,CAACuB,WACd3B,QAAOM,QAAQY;;mBAEJU,IAAIC,OAAOF,MAAAA,CAAAA;wBACNC,IAAIE,YAAYH,MAAAA,CAAAA;;SAE/B,CAAA,GAEH3B,QAAOa,IAAI,CAACd,aAAaA,SAAQwB,SAAS,IAAI;IAAC;OAAyBxB;IAASyB,KAAK,MAAA,IAAUC,MAAAA,CAAAA;AAGlG,SAAO,OAAOtB,KACZH,QAAOM,QAAQ;IAACT;IAAQK;IAAewB;IAAYX,OAAO,CAACgB,QAAuBA,QAAQN,MAAAA,CAAAA,GAC1FzB,QAAOa,IAAI,CAACmB,UAAUA,MAAMR,KAAK,MAAA,CAAA,CAAA;AAErC,CAAA;AAOK,IAAMS,mBAAmB,CAAC,EAAEC,QAAQC,UAAU,CAAA,EAAE,MACrDnC,QAAOC,IAAI,aAAA;AACT,QAAMmC,SAA6B,CAAA;AAGnC,QAAMC,uBAAuB,OAAOrC,QAAOsC,cAAcC,oBAAAA;AACzD,MAAIvB,OAAOC,OAAOoB,oBAAAA,GAAuB;AACvC,UAAMG,WAAWC,qBAAqBN,OAAAA;AACtC,UAAMO,eAAe,OAAO1C,QAAO2C,WAAW;MAC5CC,KAAK,MACHP,qBAAqBhB,MAAMwB,QACzB;WAAIL,SAASM,QAAO;QAAIjC,IAAI,CAAC,CAACkC,IAAIC,OAAAA,OAAc;QAAED;QAAIE,aAAaD;MAAQ,EAAA,CAAA;MAE/EE,OAAOC,iBAAiBC,KAAK;QAAEC,SAAS;MAAiC,CAAA;IAC3E,CAAA;AAEAC,QAAI,WAAW;MAAEZ;MAAcF;IAAS,GAAA;;;;;;AACxC,eAAW,CAACO,IAAI,EAAEC,QAAO,CAAE,KAAK;SAAIN,aAAaI,QAAO;OAAK;AAC3D,UAAIS,cAAcC,OAAOR,SAASR,SAASiB,IAAIV,EAAAA,CAAAA,GAAO;AACpDL,qBAAagB,OAAOX,EAAAA;AACpB;MACF;AAEAX,aAAOuB,KAAK;QAAEC,MAAM;QAAUC,UAAUd;QAAIC;MAAQ,CAAA;IACtD;AAEA,QAAIN,aAAaoB,OAAO,GAAG;AACzB1B,aAAOuB,KAAKI,0BAA0BrB,YAAAA,CAAAA;IACxC;EACF;AAEA,SAAOd,IAAIoC,KAAKC,SAASC,SAAS;IAChCC,UAAS,oBAAIC,KAAAA,GAAOC,YAAW;IAC/BC,QAAQ;MAAEC,MAAM;IAAO;IACvBnC,QAAQ;SAAIA;MAAQ;QAAEwB,MAAM;QAAQY,MAAMtC;MAAO;;EACnD,CAAA;AACF,CAAA;AAEF,IAAMO,uBAAuB,CAACgC,aAAAA;AAC5B,QAAMC,cAAc,oBAAIC,IAAAA;AACxB,aAAWtB,WAAWoB,UAAU;AAC9B,eAAWG,SAASvB,QAAQjB,QAAQ;AAClC,UAAIwC,MAAMhB,SAAS,UAAU;AAC3Bc,oBAAYG,IAAID,MAAMf,UAAUe,MAAM5B,OAAO;MAC/C;IACF;EACF;AAEA,SAAO0B;AACT;AAEA,IAAMX,4BAA4B,CAChCrB,iBAAAA;AAEA,SAAO;IACLkB,MAAM;;IAENkB,aAAa;IACbN,MAAMtD;;QAEF;SAAIwB,aAAaI,QAAO;MACvBjC,IAAI,CAAC,CAACkC,IAAI,EAAEgC,KAAI,CAAE,MAAM,yBAAyBhC,EAAAA,KAAOgC,OAAO;EAAKA,IAAAA,KAAS,EAAA,qBAAuB,EACpGvD,KAAK,IAAA,CAAA;;EAEZ;AACF;;;;;;;;;;;;;;;;;AD/EO,IAAMwD,YAAN,MAAMA;;;;;EA0HX,MAAMC,cACJC,SACAC,UACgC;AAChC,WAAOC,KAAAA;EAOT;EA1HA,YAA6BD,WAA6B,CAAC,GAAG;;AAT9D,qBAAA,MAAiBE,cAAjB,MAAA;AAIA,qBAAA,MAAQC,YAAR,MAAA;AAGA,qBAAA,MAAQC,YAAR,MAAA;AAIAC,qBAAAA,MAAAA,OAAAA,MAAAA;SAF6BL,WAAAA;SATZE,aAAaI,QAAOC,QAAQD,QAAOE,cAAc,CAAA,CAAA;SAI1DL,WAA+B,CAAA;SAG/BC,WAA+B,CAAA;SAIvCC,MAAM,CAAyC,EAC7CI,QACAC,QAAQC,gBACRC,UAAU,CAAA,GACVC,UAAU,CAAA,GACVC,aAAa,CAAA,GACbC,SACAC,WAAWC,mBAAmBC,KAAI,EAAE,MAEpCZ,QAAOa,IAAI,MAAM,aAAA;AACf,WAAKhB,WAAW;WAAIS;;AACpB,WAAKR,WAAW,CAAA;AAChB,YAAMgB,UAAU,KAAKhB;AAErB,YAAMiB,gBAAgBf,QAAOgB,WAAW,WAAWC,SAAyB;AAC1EH,gBAAQI,KAAKD,OAAAA;AACb,eAAOP,SAASS,UAAUF,OAAAA;AAC1B,eAAOG,eAAeC,uBAAuBJ,OAAAA;AAC7C,eAAOA;MACT,CAAA;AAIA,YAAMK,gBAAgB,OAAOP,cAAc,OAAOQ,iBAAiB;QAAEpB;QAAQG;MAAQ,CAAA,CAAC;AAGtF,YAAMF,SAAS,OAAOoB,mBAAmB;QAAEpB,QAAQC;QAAgBG;QAAYD;MAAQ,CAAA;AAGvF,SAAG;AACDkB,QAAAA,KAAI,WAAW;UACbtB,QAAQmB;UACRlB,QAAQ;YAAEsB,SAASC,cAAcvB,MAAAA;YAASwB,QAAQxB,OAAOwB;UAAO;UAChEd,SAAS,KAAKhB,SAAS8B;UACvBtB,SAAS,KAAKT,SAAS+B;UACvBrB,SAASA,SAASqB,UAAU;QAC9B,GAAA;;;;;;AAEA,cAAMzB,UAAS,OAAO0B,eAAeC,iBAAiB;aAAI,KAAKjC;aAAa,KAAKC;WAAW;UAAEM;QAAO,CAAA;AAIrG,cAAM2B,SAAS,OAAOC,cAAcC,WAAW;UAC7C9B,QAAAA;UACAM;UACAyB,2BAA2B;QAC7B,CAAA,EAAGC;;UAEDC,SAASC,cAAc;YACrBC,SAAS,MAAM5B,SAAS4B,QAAO;YAC/BC,SAAS,CAACC,UAAU9B,SAAS6B,QAAQC,KAAAA;YACrCC,QAAQ,CAACC,SAAShC,SAAS+B,OAAOC,IAAAA;YAClCC,OAAO,CAACC,YAAYlC,SAASiC,MAAMC,OAAAA;UACrC,CAAA;UACAC,OAAOC;UACP9C,QAAO+C,IAAIC,MAAMC,OAAO;QAAA;AAI1B,cAAMC,WAAW,OAAOnC,cACtBoC,KAAIC,KAAKC,UAASC,SAAS;UACzBC,UAAS,oBAAIC,KAAAA,GAAOC,YAAW;UAC/BC,QAAQ;YAAEC,MAAM;UAAY;UAC5B5B;QACF,CAAA,CAAA;AAIF,cAAM6B,YAAYC,aAAaX,QAAAA;AAC/B,YAAIU,UAAUhC,WAAW,GAAG;AAC1B;QACF,WAAW,CAACnB,SAAS;AACnB,gBAAM,IAAIqD,MAAM,qBAAA;QAClB;AAIA,cAAMC,cAAc,OAAO/D,QAAOgE,QAAQJ,WAAW,CAACK,aAAAA;AACpD,iBAAOC,SAASzD,SAASwD,QAAAA,EAAU9B,KACjCnC,QAAOmE,QACL/C,eAAegD,cAAc,CAACC,aAAa;YACzC,GAAGA;YACHC,eAAepB,SAASqB;YACxBC,YAAYP,SAASO;UACvB,EAAA,CAAA,CAAA;QAGN,CAAA;AAKA,eAAOzD,cACLoC,KAAIC,KAAKC,UAASC,SAAS;UACzBC,UAAS,oBAAIC,KAAAA,GAAOC,YAAW;UAC/BC,QAAQ;YAAEC,MAAM;UAAO;UACvB5B,QAAQgC;QACV,CAAA,CAAA;MAEJ,SAAS;AAETtC,MAAAA,KAAI,QAAQ;QAAEX,SAAS,KAAKhB,SAAS8B;MAAO,GAAA;;;;;;AAC5C,aAAO,KAAK9B;IACd,CAAA,EAAGqC,KAAK,KAAKvC,WAAW6E,YAAY,CAAA,GAAIzE,QAAO0E,SAAS,eAAA,CAAA;EAzGK;AA2HjE;AAEA,IAAM/C,gBAAgB,CAACgD,MAAcC,MAAM,OACzCD,KAAK/C,UAAUgD,MAAM,IAAID,OAAO;EAACA,KAAKE,MAAM,GAAGD,GAAAA;EAAM;EAAOD,KAAKE,MAAM,CAACD,GAAAA;EAAME,KAAK,EAAA;;;AEhMrF,SAAoBC,eAAe;AACnC,SAAuBC,UAAAA,eAAc;AAErC,SAAmCC,sBAAmCC,2BAA2B;AAEjG,SAASC,gBAAgB;AAWlB,IAAMC,gBAAgB,CAA8C,EACzEC,SAASC,cACTC,UAAU,CAAA,GACVC,aAAa,CAAA,EAAE,MAMfC,QAAOC,IAAI,aAAA;AACT,QAAMC,mBAAmB,OAAOC,oBAAoBC,eAAe;OAC9DL,WAAWM,QAAQ,CAAC,EAAEC,MAAK,MAAOA,KAAAA;OAClCR;GACJ;AAED,QAAMS,0BAAkE,OAAOL,iBAAiBM,UAC9FC,qBAAqBC,YAAYR,gBAAAA,CAAAA;AAGnC,QAAMN,UAAUe,QAAQC,MAAK,GAAI;IAACf;IAAcK;IAAkBW,OAAOC,QAAAA,CAAAA;AACzE,SAAO,OAAOlB,QAAQmB,KAAKf,QAAOgB,QAAQT,uBAAAA,CAAAA;AAK5C,CAAA;;;ACzCF,SAASU,cAAc;AAGvB,SAASC,kBAAkB;AAC3B,SAA6BC,gBAAgB;;;;;;;;;;;;;;;AAI7C,IAAMC,sBAAsBC,OAAOC;AAEnC,IAAMC,mBAAmBF,OAAOG,OAAO;;EAErCC,UAAUC;;EAEVC,SAASP;AACX,CAAA;AAQO,IAAMQ,cAAAA,SAIT,cAAcL,iBAAAA;EAEhB,OAAOM,WAAWC,QAAmC;AACnD,WAAOF,WAAWG,KAAK;MACrBN,UAAUK,OAAOE;MACjBL,SAASM,WAAWH,MAAAA;IACtB,CAAA;EACF;EAEA,OAAOI,YAAYC,KAAmC;AACpD,WAAO;MACLC,MAAM;MACNC,aAAaT,WAAWU;MACxBC,MAAMC,KAAKC,UAAUN,GAAAA;IACvB;EACF;AACF,GAfEO,kBAAA,QAAgBJ,eAAc,aAAA,GAAA;",
|
|
6
|
+
"names": ["Context", "ArtifactDiffResolver", "Context", "Tag", "Effect", "Function", "GenerationObserver", "Object", "freeze", "make", "onBegin", "Function", "constant", "Effect", "void", "onPart", "onBlock", "onEnd", "onMessage", "noop", "fromPrinter", "printer", "block", "sync", "pending", "printContentBlock", "message", "sender", "role", "printMessage", "BaseError", "AiModelError", "BaseError", "extend", "AiAssistantError", "LanguageModel", "Chunk", "Effect", "Stream", "AiParser", "AiPreprocessor", "callTool", "getToolCalls", "todo", "Obj", "TracingService", "log", "DataType", "Array", "Effect", "Option", "pipe", "Template", "Obj", "ObjectVersion", "DatabaseService", "log", "DataType", "trim", "formatSystemPrompt", "system", "blueprints", "objects", "Effect", "gen", "blueprintDefs", "pipe", "forEach", "blueprint", "succeed", "instructions", "flatMap", "template", "DatabaseService", "loadOption", "source", "map", "Array", "filter", "Option", "isSome", "trim", "Template", "process", "value", "content", "length", "join", "undefined", "objectDefs", "object", "Obj", "getDXN", "getTypename", "def", "parts", "formatUserPrompt", "prompt", "history", "blocks", "artifactDiffResolver", "serviceOption", "ArtifactDiffResolver", "versions", "gatherObjectVersions", "artifactDiff", "tryPromise", "try", "resolve", "entries", "id", "version", "lastVersion", "catch", "AiAssistantError", "wrap", "message", "log", "ObjectVersion", "equals", "get", "delete", "push", "_tag", "objectId", "size", "createArtifactUpdateBlock", "make", "DataType", "Message", "created", "Date", "toISOString", "sender", "role", "text", "messages", "artifactIds", "Map", "block", "set", "disposition", "diff", "AiSession", "runStructured", "_schema", "_options", "todo", "_semaphore", "_history", "_pending", "run", "Effect", "runSync", "makeSemaphore", "prompt", "system", "systemTemplate", "history", "objects", "blueprints", "toolkit", "observer", "GenerationObserver", "noop", "gen", "pending", "submitMessage", "fnUntraced", "message", "push", "onMessage", "TracingService", "emitConverationMessage", "promptMessage", "formatUserPrompt", "formatSystemPrompt", "log", "snippet", "createSnippet", "length", "AiPreprocessor", "preprocessPrompt", "blocks", "LanguageModel", "streamText", "disableToolCallResolution", "pipe", "AiParser", "parseResponse", "onBegin", "onBlock", "block", "onPart", "part", "onEnd", "summary", "Stream", "runCollect", "map", "Chunk", "toArray", "response", "Obj", "make", "DataType", "Message", "created", "Date", "toISOString", "sender", "role", "toolCalls", "getToolCalls", "Error", "toolResults", "forEach", "toolCall", "callTool", "provide", "layerSubframe", "context", "parentMessage", "id", "toolCallId", "withPermits", "withSpan", "text", "len", "slice", "join", "Toolkit", "Effect", "ToolExecutionService", "ToolResolverService", "isTruthy", "createToolkit", "toolkit", "toolkitParam", "toolIds", "blueprints", "Effect", "gen", "blueprintToolkit", "ToolResolverService", "resolveToolkit", "flatMap", "tools", "blueprintToolkitHandler", "toContext", "ToolExecutionService", "handlersFor", "Toolkit", "merge", "filter", "isTruthy", "pipe", "provide", "Schema", "getVersion", "ObjectId", "ObjectVersionSchema", "Schema", "Unknown", "VersionPinSchema", "Struct", "objectId", "ObjectId", "version", "VersionPin", "fromObject", "object", "make", "id", "getVersion", "createBlock", "pin", "_tag", "disposition", "DISPOSITION", "data", "JSON", "stringify", "_define_property"]
|
|
7
|
+
}
|
|
@@ -8,7 +8,7 @@ var getNer = () => {
|
|
|
8
8
|
_ner = pipeline("ner", "Xenova/bert-base-NER").then((ner) => {
|
|
9
9
|
log.info("NER model is ready", void 0, {
|
|
10
10
|
F: __dxlog_file,
|
|
11
|
-
L:
|
|
11
|
+
L: 24,
|
|
12
12
|
S: void 0,
|
|
13
13
|
C: (f, a) => f(...a)
|
|
14
14
|
});
|
|
@@ -86,4 +86,4 @@ export {
|
|
|
86
86
|
createTokenGroups,
|
|
87
87
|
combineNerTokens
|
|
88
88
|
};
|
|
89
|
-
//# sourceMappingURL=chunk-
|
|
89
|
+
//# sourceMappingURL=chunk-UB26N5UF.mjs.map
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
{
|
|
2
|
+
"version": 3,
|
|
3
|
+
"sources": ["../../../src/extraction/extraction-ner-function/named-entity-recognition.ts"],
|
|
4
|
+
"sourcesContent": ["//\n// Copyright 2025 DXOS.org\n//\n\nimport {\n type TokenClassificationOutput,\n type TokenClassificationPipelineType,\n type TokenClassificationSingle,\n pipeline,\n} from '@xenova/transformers';\n\nimport { log } from '@dxos/log';\n\nlet _ner: Promise<TokenClassificationPipelineType>;\n\n/**\n * Named Entity Recognition pipeline.\n * Initializes the pipeline on first call.\n * @returns The singleton promise that resolves to a token classification pipeline.\n */\nexport const getNer = () => {\n if (!_ner) {\n _ner = pipeline('ner', 'Xenova/bert-base-NER').then((ner) => {\n log.info('NER model is ready');\n return ner;\n });\n }\n\n return _ner;\n};\n\n/**\n * Extracts entities from text using the named entity recognition pipeline and does simple grouping of tokens.\n */\nexport const extractFullEntities = async (text: string): Promise<TokenClassificationSingle[]> => {\n const ner = await getNer();\n const entities = (await ner(text)) as TokenClassificationOutput;\n const tokenGroups = createTokenGroups(entities);\n const combinedEntities = tokenGroups.map(combineNerTokens);\n return combinedEntities;\n};\n\nexport const createTokenGroups = (tokens: TokenClassificationSingle[]): TokenClassificationSingle[][] => {\n const tokenGroups: TokenClassificationSingle[][] = [];\n\n const canBeCombined = (prevToken: TokenClassificationSingle, currentToken: TokenClassificationSingle) => {\n const [currentState, currentEntity] = currentToken.entity.split('-'); // B, I, O\n const [prevState, prevEntity] = prevToken.entity.split('-'); // B, I, O\n if (prevEntity !== currentEntity) {\n return false;\n }\n if (prevState === 'O') {\n return false;\n }\n if (currentState === 'I') {\n return true;\n }\n if ((prevState === 'B' || prevState === 'I') && currentState === 'B') {\n return currentToken.word.startsWith('#');\n }\n return false;\n };\n\n for (const token of tokens) {\n if (tokenGroups.length === 0) {\n tokenGroups.push([token]);\n continue;\n }\n const lastTokenGroup = tokenGroups.at(-1)!;\n if (canBeCombined(lastTokenGroup.at(-1)!, token)) {\n lastTokenGroup.push(token);\n continue;\n }\n tokenGroups.push([token]);\n }\n\n return tokenGroups;\n};\n\nexport const combineNerTokens = (group: TokenClassificationSingle[]): TokenClassificationSingle => {\n const combinedToken: TokenClassificationSingle = {\n entity: group.at(0)!.entity.split('-')[1],\n score: group.reduce((acc, token) => acc + token.score, 0) / group.length,\n index: group.at(0)!.index,\n word: group.at(0)!.word,\n start: group.at(0)!.start,\n end: group.at(-1)!.end,\n };\n\n for (const token of group.slice(1)) {\n if (token.word.startsWith('#')) {\n combinedToken.word += token.word.replace(/^#+/, '');\n } else {\n combinedToken.word += ' ' + token.word;\n }\n }\n\n return combinedToken;\n};\n"],
|
|
5
|
+
"mappings": ";AAIA,SAIEA,gBACK;AAEP,SAASC,WAAW;;AAEpB,IAAIC;AAOG,IAAMC,SAAS,MAAA;AACpB,MAAI,CAACD,MAAM;AACTA,WAAOF,SAAS,OAAO,sBAAA,EAAwBI,KAAK,CAACC,QAAAA;AACnDJ,UAAIK,KAAK,sBAAA,QAAA;;;;;;AACT,aAAOD;IACT,CAAA;EACF;AAEA,SAAOH;AACT;AAKO,IAAMK,sBAAsB,OAAOC,SAAAA;AACxC,QAAMH,MAAM,MAAMF,OAAAA;AAClB,QAAMM,WAAY,MAAMJ,IAAIG,IAAAA;AAC5B,QAAME,cAAcC,kBAAkBF,QAAAA;AACtC,QAAMG,mBAAmBF,YAAYG,IAAIC,gBAAAA;AACzC,SAAOF;AACT;AAEO,IAAMD,oBAAoB,CAACI,WAAAA;AAChC,QAAML,cAA6C,CAAA;AAEnD,QAAMM,gBAAgB,CAACC,WAAsCC,iBAAAA;AAC3D,UAAM,CAACC,cAAcC,aAAAA,IAAiBF,aAAaG,OAAOC,MAAM,GAAA;AAChE,UAAM,CAACC,WAAWC,UAAAA,IAAcP,UAAUI,OAAOC,MAAM,GAAA;AACvD,QAAIE,eAAeJ,eAAe;AAChC,aAAO;IACT;AACA,QAAIG,cAAc,KAAK;AACrB,aAAO;IACT;AACA,QAAIJ,iBAAiB,KAAK;AACxB,aAAO;IACT;AACA,SAAKI,cAAc,OAAOA,cAAc,QAAQJ,iBAAiB,KAAK;AACpE,aAAOD,aAAaO,KAAKC,WAAW,GAAA;IACtC;AACA,WAAO;EACT;AAEA,aAAWC,SAASZ,QAAQ;AAC1B,QAAIL,YAAYkB,WAAW,GAAG;AAC5BlB,kBAAYmB,KAAK;QAACF;OAAM;AACxB;IACF;AACA,UAAMG,iBAAiBpB,YAAYqB,GAAG,EAAC;AACvC,QAAIf,cAAcc,eAAeC,GAAG,EAAC,GAAKJ,KAAAA,GAAQ;AAChDG,qBAAeD,KAAKF,KAAAA;AACpB;IACF;AACAjB,gBAAYmB,KAAK;MAACF;KAAM;EAC1B;AAEA,SAAOjB;AACT;AAEO,IAAMI,mBAAmB,CAACkB,UAAAA;AAC/B,QAAMC,gBAA2C;IAC/CZ,QAAQW,MAAMD,GAAG,CAAA,EAAIV,OAAOC,MAAM,GAAA,EAAK,CAAA;IACvCY,OAAOF,MAAMG,OAAO,CAACC,KAAKT,UAAUS,MAAMT,MAAMO,OAAO,CAAA,IAAKF,MAAMJ;IAClES,OAAOL,MAAMD,GAAG,CAAA,EAAIM;IACpBZ,MAAMO,MAAMD,GAAG,CAAA,EAAIN;IACnBa,OAAON,MAAMD,GAAG,CAAA,EAAIO;IACpBC,KAAKP,MAAMD,GAAG,EAAC,EAAIQ;EACrB;AAEA,aAAWZ,SAASK,MAAMQ,MAAM,CAAA,GAAI;AAClC,QAAIb,MAAMF,KAAKC,WAAW,GAAA,GAAM;AAC9BO,oBAAcR,QAAQE,MAAMF,KAAKgB,QAAQ,OAAO,EAAA;IAClD,OAAO;AACLR,oBAAcR,QAAQ,MAAME,MAAMF;IACpC;EACF;AAEA,SAAOQ;AACT;",
|
|
6
|
+
"names": ["pipeline", "log", "_ner", "getNer", "then", "ner", "info", "extractFullEntities", "text", "entities", "tokenGroups", "createTokenGroups", "combinedEntities", "map", "combineNerTokens", "tokens", "canBeCombined", "prevToken", "currentToken", "currentState", "currentEntity", "entity", "split", "prevState", "prevEntity", "word", "startsWith", "token", "length", "push", "lastTokenGroup", "at", "group", "combinedToken", "score", "reduce", "acc", "index", "start", "end", "slice", "replace"]
|
|
7
|
+
}
|
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
import {
|
|
2
2
|
AiSession
|
|
3
|
-
} from "../chunk-
|
|
3
|
+
} from "../chunk-ILL2CN3G.mjs";
|
|
4
4
|
import {
|
|
5
5
|
combineNerTokens,
|
|
6
6
|
createTokenGroups,
|
|
7
7
|
extractFullEntities,
|
|
8
8
|
getNer
|
|
9
|
-
} from "../chunk-
|
|
9
|
+
} from "../chunk-UB26N5UF.mjs";
|
|
10
10
|
|
|
11
11
|
// src/extraction/extraction.ts
|
|
12
12
|
import { Schema } from "effect";
|
|
@@ -55,18 +55,15 @@ var processTranscriptMessage = async (params) => {
|
|
|
55
55
|
};
|
|
56
56
|
|
|
57
57
|
// src/extraction/extraction-llm-function/extraction-llm-function.ts
|
|
58
|
-
import {
|
|
58
|
+
import { AiService } from "@dxos/ai";
|
|
59
59
|
import { Obj } from "@dxos/echo";
|
|
60
60
|
import { create } from "@dxos/echo-schema";
|
|
61
61
|
import { defineFunction } from "@dxos/functions";
|
|
62
62
|
import { DataType as DataType2 } from "@dxos/schema";
|
|
63
63
|
|
|
64
|
-
// raw-loader:./instructions.tpl?raw
|
|
65
|
-
var instructions_default = 'You are an AI assistant specialized in processing and enhancing transcripts with contextual information.\nYour task is to analyze a given transcript and extract references to the context objects.\n\nThe context and transcript are provided in the user message in `context` and `transcript` XML tags.\n\nHere are your instructions:\n\n1. Read the transcript and context information provided above.\n\n2. Process each segment of the transcript separately.\n\n3. For each segment:\n a. Analyze the content and identify relevant information from the context.\n b. Extract a number of references to the context objects that are mentioned in the segment.\n c. Output the references in the format `{ "quote": "earnings report", "id": "01JT0JP9AX0XKGZX4MV4B69VT6" }`.\n d. If there are no references, just output an empty array.\n\n4. Maintain the exact order and structure of the original transcript segments.\n\nOutput Format:\nYou have been provided with a tool that specifies output format\n\nRemember:\n- Only quote the text that is in the transcript.\n- Analyze each segment carefully before processing it.\n- Only reference objects that appear in the context and exactly by their id, If the object is not in the context, do not insert a reference. \n\nNow, process the transcript and context provided below according to these instructions.';
|
|
66
|
-
|
|
67
64
|
// src/extraction/quotes.ts
|
|
68
65
|
import { Schema as Schema2 } from "effect";
|
|
69
|
-
import {
|
|
66
|
+
import { Filter, Query } from "@dxos/echo";
|
|
70
67
|
import { ObjectId } from "@dxos/echo-schema";
|
|
71
68
|
import { DXN } from "@dxos/keys";
|
|
72
69
|
import { log as log2 } from "@dxos/log";
|
|
@@ -114,19 +111,24 @@ var insertReferences = (text, quotes) => {
|
|
|
114
111
|
continue;
|
|
115
112
|
}
|
|
116
113
|
const regex = new RegExp(quote.quote.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"), "gi");
|
|
117
|
-
text = text.replace(regex, `[${quote.quote}]
|
|
114
|
+
text = text.replace(regex, `[${quote.quote}](${DXN.fromLocalObjectId(quote.id).toString()})`);
|
|
118
115
|
}
|
|
119
116
|
return text;
|
|
120
117
|
};
|
|
121
118
|
|
|
119
|
+
// raw-loader:/__w/dxos/dxos/packages/core/assistant/src/extraction/extraction-llm-function/instructions.tpl?raw
|
|
120
|
+
var instructions_default = 'You are an AI assistant specialized in processing and enhancing transcripts with contextual information.\nYour task is to analyze a given transcript and extract references to the context objects.\n\nThe context and transcript are provided in the user message in `context` and `transcript` XML tags.\n\nHere are your instructions:\n\n1. Read the transcript and context information provided above.\n\n2. Process each segment of the transcript separately.\n\n3. For each segment:\n a. Analyze the content and identify relevant information from the context.\n b. Extract a number of references to the context objects that are mentioned in the segment.\n c. Output the references in the format `{ "quote": "earnings report", "id": "01JT0JP9AX0XKGZX4MV4B69VT6" }`.\n d. If there are no references, just output an empty array.\n\n4. Maintain the exact order and structure of the original transcript segments.\n\nOutput Format:\nYou have been provided with a tool that specifies output format\n\nRemember:\n- Only quote the text that is in the transcript.\n- Analyze each segment carefully before processing it.\n- Only reference objects that appear in the context and exactly by their id, If the object is not in the context, do not insert a reference. \n\nNow, process the transcript and context provided below according to these instructions.';
|
|
121
|
+
|
|
122
122
|
// src/extraction/extraction-llm-function/extraction-llm-function.ts
|
|
123
|
-
var
|
|
123
|
+
var extractionAnthropicFunction = defineFunction({
|
|
124
|
+
key: "dxos.org/function/extraction/extract-entities",
|
|
125
|
+
name: "Extract Entities",
|
|
124
126
|
description: "Extract entities from the transcript message and add them to the message.",
|
|
125
127
|
inputSchema: ExtractionInput,
|
|
126
128
|
outputSchema: ExtractionOutput,
|
|
127
129
|
handler: async ({ data: { message, objects }, context }) => {
|
|
128
130
|
const startTime = performance.now();
|
|
129
|
-
const ai = context.getService(AiService);
|
|
131
|
+
const ai = context.getService(AiService.AiService);
|
|
130
132
|
const session = new AiSession({
|
|
131
133
|
operationModel: "configured"
|
|
132
134
|
});
|
|
@@ -156,18 +158,17 @@ var extractionAnthropicFn = defineFunction({
|
|
|
156
158
|
],
|
|
157
159
|
artifacts: [],
|
|
158
160
|
prompt: "",
|
|
159
|
-
tools: []
|
|
160
|
-
toolResolver: new ToolRegistry([])
|
|
161
|
+
tools: []
|
|
161
162
|
});
|
|
162
163
|
return {
|
|
164
|
+
timeElapsed: performance.now() - startTime,
|
|
163
165
|
message: create(DataType2.Message, {
|
|
164
166
|
...message,
|
|
165
|
-
blocks: message.blocks.map((block
|
|
167
|
+
blocks: message.blocks.map((block) => block._tag !== "transcript" ? block : {
|
|
166
168
|
...block,
|
|
167
169
|
text: insertReferences(block.text, result)
|
|
168
170
|
})
|
|
169
|
-
})
|
|
170
|
-
timeElapsed: performance.now() - startTime
|
|
171
|
+
})
|
|
171
172
|
};
|
|
172
173
|
}
|
|
173
174
|
});
|
|
@@ -180,7 +181,9 @@ import { invariant } from "@dxos/invariant";
|
|
|
180
181
|
import { log as log3 } from "@dxos/log";
|
|
181
182
|
import { DataType as DataType3 } from "@dxos/schema";
|
|
182
183
|
var __dxlog_file3 = "/__w/dxos/dxos/packages/core/assistant/src/extraction/extraction-ner-function/extraction-ner-function.ts";
|
|
183
|
-
var
|
|
184
|
+
var extractionNerFunction = defineFunction2({
|
|
185
|
+
key: "dxos.org/function/extraction/extract-entities",
|
|
186
|
+
name: "Extract Entities",
|
|
184
187
|
description: "Extracts named entities from a text",
|
|
185
188
|
inputSchema: ExtractionInput,
|
|
186
189
|
outputSchema: ExtractionOutput,
|
|
@@ -190,21 +193,21 @@ var extractionNerFn = defineFunction2({
|
|
|
190
193
|
options
|
|
191
194
|
}, {
|
|
192
195
|
F: __dxlog_file3,
|
|
193
|
-
L:
|
|
196
|
+
L: 29,
|
|
194
197
|
S: void 0,
|
|
195
198
|
C: (f, a) => f(...a)
|
|
196
199
|
});
|
|
197
200
|
const startTime = performance.now();
|
|
198
201
|
const { db } = context.getService(DatabaseService);
|
|
199
|
-
const { extractFullEntities: extractFullEntities2 } = await import("../named-entity-recognition-
|
|
202
|
+
const { extractFullEntities: extractFullEntities2 } = await import("../named-entity-recognition-QPYHML66.mjs");
|
|
200
203
|
const entitiesPromise = Promise.all(message.blocks.map(async (block) => {
|
|
201
|
-
invariant(block._tag === "transcript" || block._tag === "text",
|
|
204
|
+
invariant(block._tag === "transcript" || block._tag === "text", void 0, {
|
|
202
205
|
F: __dxlog_file3,
|
|
203
|
-
L:
|
|
206
|
+
L: 36,
|
|
204
207
|
S: void 0,
|
|
205
208
|
A: [
|
|
206
209
|
"block._tag === 'transcript' || block._tag === 'text'",
|
|
207
|
-
"
|
|
210
|
+
""
|
|
208
211
|
]
|
|
209
212
|
});
|
|
210
213
|
return extractFullEntities2(block.text);
|
|
@@ -214,19 +217,19 @@ var extractionNerFn = defineFunction2({
|
|
|
214
217
|
entities
|
|
215
218
|
}, {
|
|
216
219
|
F: __dxlog_file3,
|
|
217
|
-
L:
|
|
220
|
+
L: 42,
|
|
218
221
|
S: void 0,
|
|
219
222
|
C: (f, a) => f(...a)
|
|
220
223
|
});
|
|
221
224
|
const quoteReferences = await findQuotes(entities.map((entity) => entity.word), db);
|
|
222
|
-
const blocksWithReferences = message.blocks.map((block
|
|
223
|
-
invariant(block._tag === "transcript" || block._tag === "text",
|
|
225
|
+
const blocksWithReferences = message.blocks.map((block) => {
|
|
226
|
+
invariant(block._tag === "transcript" || block._tag === "text", void 0, {
|
|
224
227
|
F: __dxlog_file3,
|
|
225
|
-
L:
|
|
228
|
+
L: 49,
|
|
226
229
|
S: void 0,
|
|
227
230
|
A: [
|
|
228
231
|
"block._tag === 'transcript' || block._tag === 'text'",
|
|
229
|
-
"
|
|
232
|
+
""
|
|
230
233
|
]
|
|
231
234
|
});
|
|
232
235
|
return {
|
|
@@ -242,7 +245,7 @@ var extractionNerFn = defineFunction2({
|
|
|
242
245
|
messageWithReferences
|
|
243
246
|
}, {
|
|
244
247
|
F: __dxlog_file3,
|
|
245
|
-
L:
|
|
248
|
+
L: 58,
|
|
246
249
|
S: void 0,
|
|
247
250
|
C: (f, a) => f(...a)
|
|
248
251
|
});
|
|
@@ -259,8 +262,8 @@ export {
|
|
|
259
262
|
combineNerTokens,
|
|
260
263
|
createTokenGroups,
|
|
261
264
|
extractFullEntities,
|
|
262
|
-
|
|
263
|
-
|
|
265
|
+
extractionAnthropicFunction,
|
|
266
|
+
extractionNerFunction,
|
|
264
267
|
findQuotes,
|
|
265
268
|
getNer,
|
|
266
269
|
insertReferences,
|