@dxos/assistant 0.8.4-main.b97322e → 0.8.4-main.bc674ce
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/lib/browser/chunk-7CDDG6LL.mjs +578 -0
- package/dist/lib/browser/chunk-7CDDG6LL.mjs.map +7 -0
- package/dist/lib/browser/chunk-J5LGTIGS.mjs +10 -0
- package/dist/lib/browser/chunk-J5LGTIGS.mjs.map +7 -0
- package/dist/lib/browser/extraction/index.mjs +225 -0
- package/dist/lib/browser/extraction/index.mjs.map +7 -0
- package/dist/lib/browser/index.mjs +723 -1704
- package/dist/lib/browser/index.mjs.map +4 -4
- package/dist/lib/browser/meta.json +1 -1
- package/dist/lib/browser/testing/index.mjs +32 -0
- package/dist/lib/browser/testing/index.mjs.map +7 -0
- package/dist/lib/node-esm/chunk-CC7ER6Q3.mjs +579 -0
- package/dist/lib/node-esm/chunk-CC7ER6Q3.mjs.map +7 -0
- package/dist/lib/node-esm/chunk-HSLMI22Q.mjs +11 -0
- package/dist/lib/node-esm/chunk-HSLMI22Q.mjs.map +7 -0
- package/dist/lib/node-esm/extraction/index.mjs +226 -0
- package/dist/lib/node-esm/extraction/index.mjs.map +7 -0
- package/dist/lib/node-esm/index.mjs +723 -1705
- package/dist/lib/node-esm/index.mjs.map +4 -4
- package/dist/lib/node-esm/meta.json +1 -1
- package/dist/lib/node-esm/testing/index.mjs +33 -0
- package/dist/lib/node-esm/testing/index.mjs.map +7 -0
- package/dist/types/src/conversation/context.d.ts +129 -0
- package/dist/types/src/conversation/context.d.ts.map +1 -0
- package/dist/types/src/conversation/context.test.d.ts +2 -0
- package/dist/types/src/conversation/context.test.d.ts.map +1 -0
- package/dist/types/src/conversation/conversation.d.ts +29 -26
- package/dist/types/src/conversation/conversation.d.ts.map +1 -1
- package/dist/types/src/conversation/conversation.test.d.ts +2 -0
- package/dist/types/src/conversation/conversation.test.d.ts.map +1 -0
- package/dist/types/src/conversation/index.d.ts +1 -0
- package/dist/types/src/conversation/index.d.ts.map +1 -1
- package/dist/types/src/diff.d.ts +15 -0
- package/dist/types/src/diff.d.ts.map +1 -0
- package/dist/types/src/diff.test.d.ts +2 -0
- package/dist/types/src/diff.test.d.ts.map +1 -0
- package/dist/types/src/errors.d.ts +47 -18
- package/dist/types/src/errors.d.ts.map +1 -1
- package/dist/types/src/execution-graph.d.ts +85 -0
- package/dist/types/src/execution-graph.d.ts.map +1 -0
- package/dist/types/src/execution-graph.test.d.ts +2 -0
- package/dist/types/src/execution-graph.test.d.ts.map +1 -0
- package/dist/types/src/extraction/extraction-llm-function/extraction-llm-function.d.ts +1 -1
- package/dist/types/src/extraction/extraction-llm-function/extraction-llm-function.d.ts.map +1 -1
- package/dist/types/src/extraction/extraction-ner-function/extraction-ner-function.d.ts +258 -213
- package/dist/types/src/extraction/extraction-ner-function/extraction-ner-function.d.ts.map +1 -1
- package/dist/types/src/extraction/extraction-ner-function/named-entity-recognition.d.ts.map +1 -1
- package/dist/types/src/extraction/extraction.d.ts +345 -270
- package/dist/types/src/extraction/extraction.d.ts.map +1 -1
- package/dist/types/src/extraction/quotes.d.ts +3 -3
- package/dist/types/src/extraction/quotes.d.ts.map +1 -1
- package/dist/types/src/functions/services.d.ts +18 -4
- package/dist/types/src/functions/services.d.ts.map +1 -1
- package/dist/types/src/functions.test.d.ts +2 -0
- package/dist/types/src/functions.test.d.ts.map +1 -0
- package/dist/types/src/index.d.ts +3 -3
- package/dist/types/src/index.d.ts.map +1 -1
- package/dist/types/src/session/GenericToolkit.d.ts +85 -0
- package/dist/types/src/session/GenericToolkit.d.ts.map +1 -0
- package/dist/types/src/session/artifact-diff.d.ts +27 -0
- package/dist/types/src/session/artifact-diff.d.ts.map +1 -0
- package/dist/types/src/session/error-handling.d.ts +4 -0
- package/dist/types/src/session/error-handling.d.ts.map +1 -0
- package/dist/types/src/session/format.d.ts +12 -0
- package/dist/types/src/session/format.d.ts.map +1 -0
- package/dist/types/src/session/format.test.d.ts +2 -0
- package/dist/types/src/session/format.test.d.ts.map +1 -0
- package/dist/types/src/session/index.d.ts +4 -0
- package/dist/types/src/session/index.d.ts.map +1 -1
- package/dist/types/src/session/observer.d.ts +23 -0
- package/dist/types/src/session/observer.d.ts.map +1 -0
- package/dist/types/src/session/session.d.ts +33 -237
- package/dist/types/src/session/session.d.ts.map +1 -1
- package/dist/types/src/session/toolkit.d.ts +14 -0
- package/dist/types/src/session/toolkit.d.ts.map +1 -0
- package/dist/types/src/session/version-pin.d.ts +4 -4
- package/dist/types/src/session/version-pin.d.ts.map +1 -1
- package/dist/types/src/templates/index.d.ts +1 -1
- package/dist/types/src/templates/system.d.ts +18 -0
- package/dist/types/src/templates/system.d.ts.map +1 -0
- package/dist/types/src/testing/index.d.ts +1 -2
- package/dist/types/src/testing/index.d.ts.map +1 -1
- package/dist/types/src/testing/layer.d.ts +21 -0
- package/dist/types/src/testing/layer.d.ts.map +1 -0
- package/dist/types/src/types/artifact.d.ts +28 -0
- package/dist/types/src/types/artifact.d.ts.map +1 -0
- package/dist/types/src/types/index.d.ts +2 -0
- package/dist/types/src/types/index.d.ts.map +1 -0
- package/dist/types/tsconfig.tsbuildinfo +1 -1
- package/package.json +56 -33
- package/src/conversation/context.test.ts +46 -0
- package/src/conversation/context.ts +303 -0
- package/src/conversation/conversation.test.ts +65 -0
- package/src/conversation/conversation.ts +93 -64
- package/src/conversation/index.ts +1 -0
- package/src/diff.test.ts +43 -0
- package/src/diff.ts +84 -0
- package/src/errors.ts +6 -1
- package/src/execution-graph.test.ts +740 -0
- package/src/execution-graph.ts +509 -0
- package/src/extraction/extraction-llm-function/extraction-llm-function.test.ts +10 -9
- package/src/extraction/extraction-llm-function/extraction-llm-function.ts +48 -56
- package/src/extraction/extraction-ner-function/extraction-ner-function.test.ts +22 -21
- package/src/extraction/extraction-ner-function/extraction-ner-function.ts +42 -43
- package/src/extraction/extraction-ner-function/named-entity-recognition.test.ts +12 -6
- package/src/extraction/extraction-ner-function/named-entity-recognition.ts +2 -0
- package/src/extraction/extraction.ts +10 -9
- package/src/extraction/quotes.test.ts +3 -3
- package/src/extraction/quotes.ts +6 -7
- package/src/functions/services.ts +138 -97
- package/src/functions.conversations.json +1 -0
- package/src/functions.test.ts +74 -0
- package/src/index.ts +3 -3
- package/src/session/GenericToolkit.ts +131 -0
- package/src/session/artifact-diff.ts +41 -0
- package/src/session/error-handling.ts +41 -0
- package/src/session/format.test.ts +124 -0
- package/src/session/format.ts +143 -0
- package/src/session/index.ts +5 -0
- package/src/session/observer.ts +61 -0
- package/src/session/session.conversations.json +1 -0
- package/src/session/session.test.ts +34 -201
- package/src/session/session.ts +146 -248
- package/src/session/toolkit.ts +39 -0
- package/src/session/version-pin.ts +8 -8
- package/src/templates/index.ts +1 -1
- package/src/templates/instructions/format.tpl +61 -0
- package/src/templates/instructions/system.tpl +48 -0
- package/src/templates/system.ts +41 -0
- package/src/testing/index.ts +2 -8
- package/src/testing/layer.ts +78 -0
- package/src/types/artifact.ts +82 -0
- package/src/types/index.ts +5 -0
- package/src/typings.d.ts +5 -0
- package/dist/types/bin/chat.d.ts +0 -2
- package/dist/types/bin/chat.d.ts.map +0 -1
- package/dist/types/src/context/binder.d.ts +0 -34
- package/dist/types/src/context/binder.d.ts.map +0 -1
- package/dist/types/src/context/binding.d.ts +0 -56
- package/dist/types/src/context/binding.d.ts.map +0 -1
- package/dist/types/src/context/index.d.ts +0 -3
- package/dist/types/src/context/index.d.ts.map +0 -1
- package/dist/types/src/research/exa.d.ts +0 -12
- package/dist/types/src/research/exa.d.ts.map +0 -1
- package/dist/types/src/research/graph.d.ts +0 -51
- package/dist/types/src/research/graph.d.ts.map +0 -1
- package/dist/types/src/research/graph.test.d.ts +0 -2
- package/dist/types/src/research/graph.test.d.ts.map +0 -1
- package/dist/types/src/research/index.d.ts +0 -4
- package/dist/types/src/research/index.d.ts.map +0 -1
- package/dist/types/src/research/research.d.ts +0 -10
- package/dist/types/src/research/research.d.ts.map +0 -1
- package/dist/types/src/research/research.test.d.ts +0 -2
- package/dist/types/src/research/research.test.d.ts.map +0 -1
- package/dist/types/src/session/version-pin.test.d.ts +0 -2
- package/dist/types/src/session/version-pin.test.d.ts.map +0 -1
- package/dist/types/src/summarization/summarization.test.d.ts +0 -2
- package/dist/types/src/summarization/summarization.test.d.ts.map +0 -1
- package/dist/types/src/templates/prompt.d.ts +0 -30
- package/dist/types/src/templates/prompt.d.ts.map +0 -1
- package/dist/types/src/testing/exa-search-1748337321991.d.ts +0 -38
- package/dist/types/src/testing/exa-search-1748337321991.d.ts.map +0 -1
- package/dist/types/src/testing/exa-search-1748337331526.d.ts +0 -37
- package/dist/types/src/testing/exa-search-1748337331526.d.ts.map +0 -1
- package/dist/types/src/testing/exa-search-1748337344119.d.ts +0 -58
- package/dist/types/src/testing/exa-search-1748337344119.d.ts.map +0 -1
- package/dist/types/src/types.d.ts +0 -13
- package/dist/types/src/types.d.ts.map +0 -1
- package/src/context/binder.ts +0 -103
- package/src/context/binding.ts +0 -32
- package/src/context/index.ts +0 -6
- package/src/research/exa.ts +0 -97
- package/src/research/graph.test.ts +0 -64
- package/src/research/graph.ts +0 -369
- package/src/research/index.ts +0 -7
- package/src/research/instructions-research.tpl +0 -98
- package/src/research/research.test.ts +0 -190
- package/src/research/research.ts +0 -87
- package/src/session/__snapshots__/session.test.ts.snap +0 -32
- package/src/session/version-pin.test.ts +0 -7
- package/src/summarization/summarization.test.ts +0 -208
- package/src/templates/assistant.tpl +0 -111
- package/src/templates/prompt.ts +0 -45
- package/src/testing/exa-search-1748337321991.ts +0 -131
- package/src/testing/exa-search-1748337331526.ts +0 -144
- package/src/testing/exa-search-1748337344119.ts +0 -133
- package/src/typedefs.d.ts +0 -8
- package/src/types.ts +0 -47
- /package/src/extraction/extraction-llm-function/{instructions.tpl → transcription-instructions.tpl} +0 -0
|
@@ -2,39 +2,42 @@
|
|
|
2
2
|
// Copyright 2025 DXOS.org
|
|
3
3
|
//
|
|
4
4
|
|
|
5
|
-
import { describe, test
|
|
5
|
+
import { beforeAll, describe, test } from 'vitest';
|
|
6
6
|
|
|
7
|
-
import { type
|
|
7
|
+
import { type Database } from '@dxos/echo';
|
|
8
8
|
import { EchoTestBuilder } from '@dxos/echo-db/testing';
|
|
9
|
-
import { FunctionExecutor, ServiceContainer } from '@dxos/functions';
|
|
9
|
+
import { FunctionExecutor, ServiceContainer } from '@dxos/functions-runtime';
|
|
10
10
|
import { log } from '@dxos/log';
|
|
11
|
-
import {
|
|
12
|
-
import {
|
|
11
|
+
import { TestSchema } from '@dxos/schema/testing';
|
|
12
|
+
import { type Message, Organization, Person } from '@dxos/types';
|
|
13
|
+
import { createTestData } from '@dxos/types/testing';
|
|
13
14
|
import { range } from '@dxos/util';
|
|
14
15
|
|
|
15
|
-
import { extractionNerFn } from './extraction-ner-function';
|
|
16
16
|
import { processTranscriptMessage } from '../extraction';
|
|
17
17
|
|
|
18
|
+
import { extractionNerFunction } from './extraction-ner-function';
|
|
19
|
+
|
|
18
20
|
describe.skip('NER EntityExtraction', () => {
|
|
19
21
|
let builder: EchoTestBuilder;
|
|
20
|
-
let db:
|
|
22
|
+
let db: Database.Database;
|
|
21
23
|
let executor: FunctionExecutor;
|
|
22
24
|
let testData: {
|
|
23
|
-
transcriptJosiah:
|
|
24
|
-
transcriptWoflram:
|
|
25
|
-
transcriptMessages:
|
|
26
|
-
documents:
|
|
27
|
-
contacts: Record<string,
|
|
28
|
-
organizations: Record<string,
|
|
25
|
+
transcriptJosiah: Message.Message[];
|
|
26
|
+
transcriptWoflram: Message.Message[];
|
|
27
|
+
transcriptMessages: Message.Message[];
|
|
28
|
+
documents: TestSchema.DocumentType[];
|
|
29
|
+
contacts: Record<string, Person.Person>;
|
|
30
|
+
organizations: Record<string, Organization.Organization>;
|
|
29
31
|
};
|
|
30
|
-
|
|
32
|
+
|
|
33
|
+
const TYPES = [Organization.Organization, Person.Person, TestSchema.Person, TestSchema.DocumentType];
|
|
31
34
|
|
|
32
35
|
beforeAll(async () => {
|
|
33
36
|
// TODO(dmaretskyi): Helper to scaffold this from a config.
|
|
34
37
|
builder = await new EchoTestBuilder().open();
|
|
35
38
|
const { db: db1 } = await builder.createDatabase({ indexing: { vector: true } });
|
|
36
39
|
db = db1;
|
|
37
|
-
db.graph.schemaRegistry.
|
|
40
|
+
await db.graph.schemaRegistry.register(TYPES);
|
|
38
41
|
const data = createTestData();
|
|
39
42
|
testData = {
|
|
40
43
|
transcriptJosiah: data.transcriptJosiah,
|
|
@@ -44,8 +47,8 @@ describe.skip('NER EntityExtraction', () => {
|
|
|
44
47
|
contacts: Object.fromEntries(Object.entries(data.contacts).map(([key, value]) => [key, db.add(value)])),
|
|
45
48
|
organizations: Object.fromEntries(Object.entries(data.organizations).map(([key, value]) => [key, db.add(value)])),
|
|
46
49
|
};
|
|
47
|
-
await db.flush();
|
|
48
50
|
|
|
51
|
+
await db.flush();
|
|
49
52
|
executor = new FunctionExecutor(
|
|
50
53
|
new ServiceContainer().setServices({
|
|
51
54
|
database: { db },
|
|
@@ -64,7 +67,7 @@ describe.skip('NER EntityExtraction', () => {
|
|
|
64
67
|
message,
|
|
65
68
|
objects: [...documents, ...Object.values(contacts)],
|
|
66
69
|
},
|
|
67
|
-
function:
|
|
70
|
+
function: extractionNerFunction,
|
|
68
71
|
executor,
|
|
69
72
|
});
|
|
70
73
|
log.info('output', enhancedMessage);
|
|
@@ -73,7 +76,6 @@ describe.skip('NER EntityExtraction', () => {
|
|
|
73
76
|
|
|
74
77
|
test('computational irreducibility', async () => {
|
|
75
78
|
const { transcriptWoflram, documents, contacts } = testData;
|
|
76
|
-
|
|
77
79
|
log.info('context', { documents, contacts });
|
|
78
80
|
const message = transcriptWoflram[0];
|
|
79
81
|
log.info('input', message);
|
|
@@ -85,7 +87,7 @@ describe.skip('NER EntityExtraction', () => {
|
|
|
85
87
|
message,
|
|
86
88
|
objects: [...documents, ...Object.values(contacts)],
|
|
87
89
|
},
|
|
88
|
-
function:
|
|
90
|
+
function: extractionNerFunction,
|
|
89
91
|
executor,
|
|
90
92
|
});
|
|
91
93
|
log.info('output', { message: enhancedMessage.blocks[0], timeElapsed });
|
|
@@ -95,7 +97,6 @@ describe.skip('NER EntityExtraction', () => {
|
|
|
95
97
|
|
|
96
98
|
test('org and document linking', async () => {
|
|
97
99
|
const { transcriptJosiah, documents, contacts, organizations } = testData;
|
|
98
|
-
|
|
99
100
|
log.info('context', { contacts, organizations, documents });
|
|
100
101
|
|
|
101
102
|
for (const message of transcriptJosiah) {
|
|
@@ -106,7 +107,7 @@ describe.skip('NER EntityExtraction', () => {
|
|
|
106
107
|
message,
|
|
107
108
|
objects: [...documents, ...Object.values(contacts), ...Object.values(organizations)],
|
|
108
109
|
},
|
|
109
|
-
function:
|
|
110
|
+
function: extractionNerFunction,
|
|
110
111
|
executor,
|
|
111
112
|
});
|
|
112
113
|
log.info('output', enhancedMessage);
|
|
@@ -2,56 +2,55 @@
|
|
|
2
2
|
// Copyright 2025 DXOS.org
|
|
3
3
|
//
|
|
4
4
|
|
|
5
|
-
|
|
6
|
-
// @ts-nocheck
|
|
5
|
+
import { defineFunction } from '@dxos/functions';
|
|
7
6
|
|
|
8
|
-
import { asyncTimeout } from '@dxos/async';
|
|
9
|
-
import { create } from '@dxos/echo-schema';
|
|
10
|
-
import { DatabaseService, defineFunction } from '@dxos/functions';
|
|
11
|
-
import { invariant } from '@dxos/invariant';
|
|
12
|
-
import { log } from '@dxos/log';
|
|
13
|
-
import { DataType } from '@dxos/schema';
|
|
14
|
-
|
|
15
|
-
import { extractFullEntities } from './named-entity-recognition';
|
|
16
7
|
import { ExtractionInput, ExtractionOutput } from '../extraction';
|
|
17
|
-
import { findQuotes, insertReferences } from '../quotes';
|
|
18
8
|
|
|
9
|
+
/**
|
|
10
|
+
* Named Entity Recognition model
|
|
11
|
+
*/
|
|
19
12
|
// TODO(mykola): Make it use vector index to query objects and not pass objects as input.
|
|
20
|
-
export const
|
|
13
|
+
export const extractionNerFunction = defineFunction({
|
|
14
|
+
key: 'dxos.org/function/extraction/extract-entities',
|
|
15
|
+
name: 'Extract Entities',
|
|
21
16
|
description: 'Extracts named entities from a text',
|
|
22
17
|
inputSchema: ExtractionInput,
|
|
23
18
|
outputSchema: ExtractionOutput,
|
|
24
19
|
handler: async ({ data: { message, options }, context }) => {
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
const
|
|
28
|
-
|
|
29
|
-
const
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
const
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
const
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
});
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
};
|
|
20
|
+
throw new Error('not implemented');
|
|
21
|
+
// log.info('input', { message, options });
|
|
22
|
+
// const startTime = performance.now();
|
|
23
|
+
// const { db } = context.getService(Database.Service);
|
|
24
|
+
// const { extractFullEntities } = await import('./named-entity-recognition');
|
|
25
|
+
|
|
26
|
+
// const entitiesPromise = Promise.all(
|
|
27
|
+
// message.blocks.map(async (block) => {
|
|
28
|
+
// invariant(block._tag === 'transcript' || block._tag === 'text');
|
|
29
|
+
// return extractFullEntities(block.text);
|
|
30
|
+
// }),
|
|
31
|
+
// ).then((entities) => entities.flat());
|
|
32
|
+
// const entities = options?.timeout ? await asyncTimeout(entitiesPromise, options.timeout) : await entitiesPromise;
|
|
33
|
+
|
|
34
|
+
// log.info('entities', { entities });
|
|
35
|
+
// const quoteReferences = await findQuotes(
|
|
36
|
+
// entities.map((entity) => entity.word),
|
|
37
|
+
// db,
|
|
38
|
+
// );
|
|
39
|
+
|
|
40
|
+
// const blocksWithReferences = message.blocks.map((block) => {
|
|
41
|
+
// invariant(block._tag === 'transcript' || block._tag === 'text');
|
|
42
|
+
// return { ...block, text: insertReferences(block._tag, quoteReferences) };
|
|
43
|
+
// });
|
|
44
|
+
|
|
45
|
+
// const messageWithReferences = create(Message.Message, {
|
|
46
|
+
// ...message,
|
|
47
|
+
// blocks: blocksWithReferences,
|
|
48
|
+
// });
|
|
49
|
+
|
|
50
|
+
// log.info('output', { messageWithReferences });
|
|
51
|
+
// return {
|
|
52
|
+
// message: messageWithReferences,
|
|
53
|
+
// timeElapsed: performance.now() - startTime,
|
|
54
|
+
// };
|
|
56
55
|
},
|
|
57
56
|
});
|
|
@@ -6,11 +6,13 @@ import { type TokenClassificationOutput, type TokenClassificationPipelineType }
|
|
|
6
6
|
import { beforeAll, describe, expect, test } from 'vitest';
|
|
7
7
|
|
|
8
8
|
import { log } from '@dxos/log';
|
|
9
|
-
import {
|
|
9
|
+
import { type ContentBlock, type Message } from '@dxos/types';
|
|
10
|
+
import { createTestData } from '@dxos/types/testing';
|
|
10
11
|
|
|
11
12
|
import { combineNerTokens, createTokenGroups, extractFullEntities, getNer } from './named-entity-recognition';
|
|
12
13
|
|
|
13
|
-
|
|
14
|
+
// TODO(mykola): This should not run on CI.
|
|
15
|
+
describe.skip('NamedEntityRecognition', () => {
|
|
14
16
|
let ner: TokenClassificationPipelineType;
|
|
15
17
|
|
|
16
18
|
beforeAll(async () => {
|
|
@@ -56,8 +58,10 @@ describe('NamedEntityRecognition', () => {
|
|
|
56
58
|
test('own data', async () => {
|
|
57
59
|
const { transcriptWoflram, transcriptJosiah } = await createTestData();
|
|
58
60
|
|
|
59
|
-
const blocks = [...transcriptWoflram, ...transcriptJosiah].flatMap((message) =>
|
|
60
|
-
message.blocks.flatMap((block) =>
|
|
61
|
+
const blocks = [...transcriptWoflram, ...transcriptJosiah].flatMap((message: Message.Message) =>
|
|
62
|
+
message.blocks.flatMap((block: ContentBlock.Any) =>
|
|
63
|
+
block._tag === 'transcript' ? [(block as ContentBlock.Transcript).text] : [],
|
|
64
|
+
),
|
|
61
65
|
);
|
|
62
66
|
|
|
63
67
|
for (const block of blocks) {
|
|
@@ -69,8 +73,10 @@ describe('NamedEntityRecognition', () => {
|
|
|
69
73
|
test('more own data', async () => {
|
|
70
74
|
const { transcriptMessages } = await createTestData();
|
|
71
75
|
|
|
72
|
-
const blocks = transcriptMessages.flatMap((message) =>
|
|
73
|
-
message.blocks.flatMap((block) =>
|
|
76
|
+
const blocks = transcriptMessages.flatMap((message: Message.Message) =>
|
|
77
|
+
message.blocks.flatMap((block: ContentBlock.Any) =>
|
|
78
|
+
block._tag === 'transcript' ? [(block as ContentBlock.Transcript).text] : [],
|
|
79
|
+
),
|
|
74
80
|
);
|
|
75
81
|
|
|
76
82
|
for (const block of blocks) {
|
|
@@ -12,6 +12,7 @@ import {
|
|
|
12
12
|
import { log } from '@dxos/log';
|
|
13
13
|
|
|
14
14
|
let _ner: Promise<TokenClassificationPipelineType>;
|
|
15
|
+
|
|
15
16
|
/**
|
|
16
17
|
* Named Entity Recognition pipeline.
|
|
17
18
|
* Initializes the pipeline on first call.
|
|
@@ -24,6 +25,7 @@ export const getNer = () => {
|
|
|
24
25
|
return ner;
|
|
25
26
|
});
|
|
26
27
|
}
|
|
28
|
+
|
|
27
29
|
return _ner;
|
|
28
30
|
};
|
|
29
31
|
|
|
@@ -2,17 +2,18 @@
|
|
|
2
2
|
// Copyright 2025 DXOS.org
|
|
3
3
|
//
|
|
4
4
|
|
|
5
|
-
import
|
|
5
|
+
import * as Schema from 'effect/Schema';
|
|
6
6
|
|
|
7
7
|
import { asyncTimeout } from '@dxos/async';
|
|
8
|
-
import {
|
|
9
|
-
import { type FunctionDefinition
|
|
8
|
+
import { Type } from '@dxos/echo';
|
|
9
|
+
import { type FunctionDefinition } from '@dxos/functions';
|
|
10
|
+
import { type FunctionExecutor } from '@dxos/functions-runtime';
|
|
10
11
|
import { log } from '@dxos/log';
|
|
11
|
-
import {
|
|
12
|
+
import { Message } from '@dxos/types';
|
|
12
13
|
|
|
13
14
|
export const ExtractionInput = Schema.Struct({
|
|
14
|
-
message:
|
|
15
|
-
objects: Schema.optional(Schema.Array(
|
|
15
|
+
message: Message.Message,
|
|
16
|
+
objects: Schema.optional(Schema.Array(Type.Obj)),
|
|
16
17
|
options: Schema.optional(
|
|
17
18
|
Schema.Struct({
|
|
18
19
|
timeout: Schema.optional(Schema.Number),
|
|
@@ -23,12 +24,12 @@ export const ExtractionInput = Schema.Struct({
|
|
|
23
24
|
export interface ExtractionInput extends Schema.Schema.Type<typeof ExtractionInput> {}
|
|
24
25
|
|
|
25
26
|
export const ExtractionOutput = Schema.Struct({
|
|
26
|
-
message:
|
|
27
|
+
message: Message.Message,
|
|
27
28
|
timeElapsed: Schema.Number,
|
|
28
29
|
});
|
|
29
30
|
export interface ExtractionOutput extends Schema.Schema.Type<typeof ExtractionOutput> {}
|
|
30
31
|
|
|
31
|
-
export type
|
|
32
|
+
export type ProcessTranscriptMessageProps = {
|
|
32
33
|
input: ExtractionInput;
|
|
33
34
|
function: FunctionDefinition<ExtractionInput, ExtractionOutput>;
|
|
34
35
|
executor: FunctionExecutor;
|
|
@@ -54,7 +55,7 @@ export type ExtractionFunction = FunctionDefinition<ExtractionInput, ExtractionO
|
|
|
54
55
|
* Extract entities from the transcript message and add them to the message.
|
|
55
56
|
*/
|
|
56
57
|
// TODO(dmaretskyi): Move context to a vector search index.
|
|
57
|
-
export const processTranscriptMessage = async (params:
|
|
58
|
+
export const processTranscriptMessage = async (params: ProcessTranscriptMessageProps): Promise<ExtractionOutput> => {
|
|
58
59
|
try {
|
|
59
60
|
if (params.options?.timeout && params.options.timeout > 0) {
|
|
60
61
|
return await asyncTimeout(params.executor.invoke(params.function, params.input), params.options.timeout);
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
// Copyright 2025 DXOS.org
|
|
3
3
|
//
|
|
4
4
|
|
|
5
|
-
import { describe,
|
|
5
|
+
import { describe, expect, test } from 'vitest';
|
|
6
6
|
|
|
7
7
|
import { insertReferences } from './quotes';
|
|
8
8
|
|
|
@@ -13,7 +13,7 @@ describe('insertReferences', () => {
|
|
|
13
13
|
};
|
|
14
14
|
|
|
15
15
|
expect(insertReferences('This is a computational irreducibility test.', quotes)).toBe(
|
|
16
|
-
'This is a [computational irreducibility]
|
|
16
|
+
'This is a [computational irreducibility](dxn:echo:@:01JTG9JW11XGWJZ32AW8ET93D1) test.',
|
|
17
17
|
);
|
|
18
18
|
expect(
|
|
19
19
|
insertReferences(
|
|
@@ -21,7 +21,7 @@ describe('insertReferences', () => {
|
|
|
21
21
|
quotes,
|
|
22
22
|
),
|
|
23
23
|
).toBe(
|
|
24
|
-
"And what I'd like to talk today about is Steven Wolfram's concept of a [computational irreducibility]
|
|
24
|
+
"And what I'd like to talk today about is Steven Wolfram's concept of a [computational irreducibility](dxn:echo:@:01JTG9JW11XGWJZ32AW8ET93D1).",
|
|
25
25
|
);
|
|
26
26
|
});
|
|
27
27
|
});
|
package/src/extraction/quotes.ts
CHANGED
|
@@ -2,11 +2,10 @@
|
|
|
2
2
|
// Copyright 2025 DXOS.org
|
|
3
3
|
//
|
|
4
4
|
|
|
5
|
-
import
|
|
5
|
+
import * as Schema from 'effect/Schema';
|
|
6
6
|
|
|
7
|
-
import {
|
|
8
|
-
import {
|
|
9
|
-
import { ObjectId } from '@dxos/echo-schema';
|
|
7
|
+
import { type Database, Filter, Query } from '@dxos/echo';
|
|
8
|
+
import { ObjectId } from '@dxos/keys';
|
|
10
9
|
import { DXN } from '@dxos/keys';
|
|
11
10
|
import { log } from '@dxos/log';
|
|
12
11
|
|
|
@@ -28,10 +27,10 @@ export const ReferencedQuotes = Schema.Struct({
|
|
|
28
27
|
});
|
|
29
28
|
export type ReferencedQuotes = Schema.Schema.Type<typeof ReferencedQuotes>;
|
|
30
29
|
|
|
31
|
-
export const findQuotes = async (quotes: string[], db:
|
|
30
|
+
export const findQuotes = async (quotes: string[], db: Database.Database): Promise<ReferencedQuotes> => {
|
|
32
31
|
const references = await Promise.all(
|
|
33
32
|
quotes.map(async (quote) => {
|
|
34
|
-
const
|
|
33
|
+
const objects = await db.query(Query.select(Filter.text(quote, { type: 'vector' }))).run();
|
|
35
34
|
log.info('found quotes in vector index', { objects, quote });
|
|
36
35
|
return objects.length > 0 ? [{ id: objects[0].id.toString(), quote }] : [];
|
|
37
36
|
}),
|
|
@@ -51,7 +50,7 @@ export const insertReferences = (text: string, quotes: ReferencedQuotes) => {
|
|
|
51
50
|
|
|
52
51
|
// Use a case-insensitive regular expression to replace the quote.
|
|
53
52
|
const regex = new RegExp(quote.quote.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'), 'gi');
|
|
54
|
-
text = text.replace(regex, `[${quote.quote}]
|
|
53
|
+
text = text.replace(regex, `[${quote.quote}](${DXN.fromLocalObjectId(quote.id).toString()})`);
|
|
55
54
|
}
|
|
56
55
|
|
|
57
56
|
return text;
|
|
@@ -2,72 +2,140 @@
|
|
|
2
2
|
// Copyright 2025 DXOS.org
|
|
3
3
|
//
|
|
4
4
|
|
|
5
|
-
import
|
|
6
|
-
import
|
|
5
|
+
import * as Tool from '@effect/ai/Tool';
|
|
6
|
+
import type * as Toolkit from '@effect/ai/Toolkit';
|
|
7
|
+
import * as Context from 'effect/Context';
|
|
8
|
+
import * as Effect from 'effect/Effect';
|
|
9
|
+
import * as Layer from 'effect/Layer';
|
|
10
|
+
import * as Record from 'effect/Record';
|
|
11
|
+
import * as Schema from 'effect/Schema';
|
|
12
|
+
import type * as SchemaAST from 'effect/SchemaAST';
|
|
7
13
|
|
|
8
14
|
import { AiToolNotFoundError, ToolExecutionService, ToolResolverService } from '@dxos/ai';
|
|
9
15
|
import { todo } from '@dxos/debug';
|
|
10
|
-
import {
|
|
16
|
+
import { Query, Type } from '@dxos/echo';
|
|
17
|
+
import { Database } from '@dxos/echo';
|
|
18
|
+
import { Function, FunctionDefinition, FunctionInvocationService } from '@dxos/functions';
|
|
11
19
|
import { invariant } from '@dxos/invariant';
|
|
12
20
|
|
|
21
|
+
import { RefFromLLM } from '../types';
|
|
22
|
+
|
|
23
|
+
/**
|
|
24
|
+
* Constructs a `ToolResolverService` whose `resolve(id)` looks up tools in the following order:
|
|
25
|
+
* 1. Toolkit: return an existing tool from the provided `toolkit` if one is present under `id`.
|
|
26
|
+
* 2. Functions in DB: query `FunctionType` by `key=id`; if found, deserialize and project to a tool
|
|
27
|
+
* (propagating any `deployedFunctionId` discovered in the object's metadata).
|
|
28
|
+
* 3. Functions passed in: fall back to a matching `FunctionDefinition` from the `functions` array.
|
|
29
|
+
*
|
|
30
|
+
* If none of the above yield a match, the effect fails with `AiToolNotFoundError`.
|
|
31
|
+
*
|
|
32
|
+
* Requires `Database.Service` in the environment.
|
|
33
|
+
*/
|
|
13
34
|
export const makeToolResolverFromFunctions = (
|
|
14
|
-
functions: FunctionDefinition
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
35
|
+
functions: FunctionDefinition.Any[],
|
|
36
|
+
toolkit: Toolkit.Toolkit<any>,
|
|
37
|
+
): Layer.Layer<ToolResolverService, never, Database.Service> => {
|
|
38
|
+
return Layer.effect(
|
|
39
|
+
ToolResolverService,
|
|
40
|
+
Effect.gen(function* () {
|
|
41
|
+
const dbService = yield* Database.Service;
|
|
42
|
+
return {
|
|
43
|
+
resolve: (id): Effect.Effect<Tool.Any, AiToolNotFoundError> =>
|
|
44
|
+
Effect.gen(function* () {
|
|
45
|
+
// TODO(dmaretskyi): Use FunctionInvocationService.resolveFunction().
|
|
46
|
+
|
|
47
|
+
const tool = toolkit.tools[id];
|
|
48
|
+
if (tool) {
|
|
49
|
+
return tool;
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
const [dbFunction] = yield* Database.Service.runQuery(Query.type(Function.Function, { key: id }));
|
|
53
|
+
|
|
54
|
+
const functionDef = dbFunction
|
|
55
|
+
? FunctionDefinition.deserialize(dbFunction)
|
|
56
|
+
: functions.find((fn) => fn.key === id);
|
|
57
|
+
|
|
58
|
+
if (!functionDef) {
|
|
59
|
+
return yield* Effect.fail(new AiToolNotFoundError(id));
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
return projectFunctionToTool(functionDef);
|
|
63
|
+
}).pipe(Effect.provideService(Database.Service, dbService)),
|
|
64
|
+
} satisfies Context.Tag.Service<ToolResolverService>;
|
|
24
65
|
}),
|
|
25
|
-
|
|
66
|
+
);
|
|
26
67
|
};
|
|
27
68
|
|
|
28
69
|
export const makeToolExecutionServiceFromFunctions = (
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
70
|
+
toolkit: Toolkit.Toolkit<any>,
|
|
71
|
+
handlersLayer: Layer.Layer<Tool.Handler<any>, never, never>,
|
|
72
|
+
): Layer.Layer<ToolExecutionService, never, FunctionInvocationService> => {
|
|
73
|
+
return Layer.effect(
|
|
74
|
+
ToolExecutionService,
|
|
75
|
+
Effect.gen(function* () {
|
|
76
|
+
const toolkitHandler = yield* toolkit.pipe(Effect.provide(handlersLayer));
|
|
77
|
+
invariant(isHandlerLike(toolkitHandler));
|
|
78
|
+
const functionInvocationService = yield* FunctionInvocationService;
|
|
79
|
+
|
|
80
|
+
return {
|
|
81
|
+
handlersFor: (toolkit) => {
|
|
82
|
+
const makeHandler = (tool: Tool.Any): ((params: unknown) => Effect.Effect<unknown, any, any>) => {
|
|
83
|
+
return Effect.fn(`toolFunctionHandler ${tool.name}`)(function* (input: any) {
|
|
84
|
+
if (toolkitHandler.tools[tool.name]) {
|
|
85
|
+
if (Tool.isProviderDefined(tool)) {
|
|
86
|
+
throw new Error('Attempted to call a provider-defined tool');
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
// TODO(wittjosiah): Everything is `never` here.
|
|
90
|
+
const { result } = yield* (toolkitHandler.handle as any)(tool.name, input);
|
|
91
|
+
return result;
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
const { definition: functionDef } = Context.get(FunctionToolAnnotation)(tool.annotations as any);
|
|
95
|
+
|
|
96
|
+
return yield* functionInvocationService
|
|
97
|
+
.invokeFunction(functionDef, input as any)
|
|
98
|
+
.pipe(Effect.catchAllDefect((defect) => Effect.fail(defect)));
|
|
99
|
+
});
|
|
100
|
+
};
|
|
101
|
+
|
|
102
|
+
return toolkit.of(
|
|
103
|
+
Record.map(toolkit.tools, (tool, _name) => (Tool.isUserDefined(tool) ? makeHandler(tool) : null)) as any,
|
|
104
|
+
) as any;
|
|
105
|
+
},
|
|
43
106
|
};
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
},
|
|
47
|
-
});
|
|
107
|
+
}),
|
|
108
|
+
);
|
|
48
109
|
};
|
|
49
110
|
|
|
50
|
-
|
|
51
|
-
* Tools that are projected from functions have this annotation.
|
|
52
|
-
*/
|
|
53
|
-
class FunctionToolAnnotation extends Context.Tag('@dxos/assisatnt/FunctionToolAnnotation')<
|
|
111
|
+
class FunctionToolAnnotation extends Context.Tag('@dxos/assistant/FunctionToolAnnotation')<
|
|
54
112
|
FunctionToolAnnotation,
|
|
55
|
-
{
|
|
56
|
-
functionName: string;
|
|
57
|
-
}
|
|
113
|
+
{ definition: FunctionDefinition.Any }
|
|
58
114
|
>() {}
|
|
59
115
|
|
|
60
|
-
const toolCache = new WeakMap<FunctionDefinition
|
|
61
|
-
|
|
116
|
+
const toolCache = new WeakMap<FunctionDefinition.Any, Tool.Any>();
|
|
117
|
+
|
|
118
|
+
/**
|
|
119
|
+
* Projects a `FunctionDefinition` into an `AiTool`.
|
|
120
|
+
*
|
|
121
|
+
* @param fn The function definition to project into a tool.
|
|
122
|
+
* @param meta Optional projection metadata.
|
|
123
|
+
* @param meta.deployedFunctionId Backend deployment ID used for remote invocation when present.
|
|
124
|
+
* This is the EDGE service's function deployment identifier (not the ECHO object ID/DXN and not `FunctionDefinition.key`).
|
|
125
|
+
*/
|
|
126
|
+
const projectFunctionToTool = (fn: FunctionDefinition.Any): Tool.Any => {
|
|
62
127
|
if (toolCache.has(fn)) {
|
|
63
128
|
return toolCache.get(fn)!;
|
|
64
129
|
}
|
|
65
|
-
|
|
130
|
+
|
|
131
|
+
// TODO(burdon): Use and map function.key?
|
|
132
|
+
// TODO(dmaretskyi): Use function key instead.
|
|
133
|
+
const tool = Tool.make(makeToolName(fn.name), {
|
|
66
134
|
description: fn.description,
|
|
67
135
|
parameters: createStructFieldsFromSchema(fn.inputSchema),
|
|
68
136
|
// TODO(dmaretskyi): Include output schema.
|
|
69
137
|
failure: Schema.Any, // TODO(dmaretskyi): Better type for the failure?
|
|
70
|
-
}).annotate(FunctionToolAnnotation, {
|
|
138
|
+
}).annotate(FunctionToolAnnotation, { definition: fn });
|
|
71
139
|
toolCache.set(fn, tool);
|
|
72
140
|
return tool;
|
|
73
141
|
};
|
|
@@ -76,64 +144,37 @@ const projectFunctionToTool = (fn: FunctionDefinition<any, any>): AiTool.Any =>
|
|
|
76
144
|
* @returns Tool name produced from function name by escaping invalid characters.
|
|
77
145
|
*/
|
|
78
146
|
const makeToolName = (name: string) => {
|
|
79
|
-
const toolName = name.replace(/[^a-zA-Z0-9]/g, '');
|
|
80
|
-
invariant(toolName.match(/^[a-
|
|
147
|
+
const toolName = name.toLowerCase().replace(/[^a-zA-Z0-9]/g, '-');
|
|
148
|
+
invariant(toolName.match(/^[a-z_][a-z0-9-_]*$/));
|
|
81
149
|
return toolName;
|
|
82
150
|
};
|
|
83
151
|
|
|
84
152
|
// TODO(dmaretskyi): Factor out.
|
|
85
153
|
const createStructFieldsFromSchema = (schema: Schema.Schema<any, any>): Record<string, Schema.Schema<any, any>> => {
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
return Object.fromEntries(
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
154
|
+
switch (schema.ast._tag) {
|
|
155
|
+
case 'TypeLiteral':
|
|
156
|
+
return Object.fromEntries(
|
|
157
|
+
schema.ast.propertySignatures.map((prop) => [prop.name, Schema.make(mapSchemaTypeForLLM(prop.type))]),
|
|
158
|
+
);
|
|
159
|
+
case 'VoidKeyword':
|
|
160
|
+
return {};
|
|
161
|
+
default:
|
|
162
|
+
return todo(`Unsupported schema AST: ${schema.ast._tag}`);
|
|
163
|
+
}
|
|
92
164
|
};
|
|
93
165
|
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
},
|
|
110
|
-
};
|
|
111
|
-
|
|
112
|
-
// TODO(dmaretskyi): This should be delegated to a function invoker service.
|
|
113
|
-
const data = yield* Effect.gen(function* () {
|
|
114
|
-
const result = fnDef.handler({ context, data: input });
|
|
115
|
-
if (Effect.isEffect(result)) {
|
|
116
|
-
return yield* (result as Effect.Effect<unknown, unknown, Services>).pipe(Effect.orDie);
|
|
117
|
-
} else if (
|
|
118
|
-
typeof result === 'object' &&
|
|
119
|
-
result !== null &&
|
|
120
|
-
'then' in result &&
|
|
121
|
-
typeof result.then === 'function'
|
|
122
|
-
) {
|
|
123
|
-
return yield* Effect.promise(() => result);
|
|
124
|
-
} else {
|
|
125
|
-
return result;
|
|
126
|
-
}
|
|
127
|
-
}).pipe(
|
|
128
|
-
Effect.orDie,
|
|
129
|
-
Effect.catchAllDefect((defect) =>
|
|
130
|
-
Effect.die(new FunctionError('Error running function', { context: { name: fnDef.name }, cause: defect })),
|
|
131
|
-
),
|
|
132
|
-
);
|
|
133
|
-
|
|
134
|
-
// Assert output matches schema
|
|
135
|
-
const assertOutput = fnDef.outputSchema?.pipe(Schema.asserts);
|
|
136
|
-
(assertOutput as any)(data);
|
|
137
|
-
|
|
138
|
-
return data;
|
|
139
|
-
}).pipe(Effect.withSpan('invokeFunction', { attributes: { name: fnDef.name } }));
|
|
166
|
+
/**
|
|
167
|
+
* Picks an LLM-friendly schema type for the given schema AST.
|
|
168
|
+
* The picked schema type decodes to the original schema type.
|
|
169
|
+
*/
|
|
170
|
+
const mapSchemaTypeForLLM = (ast: SchemaAST.AST): SchemaAST.AST => {
|
|
171
|
+
if (Type.Ref.isRefSchemaAST(ast)) {
|
|
172
|
+
return RefFromLLM.ast;
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
return ast;
|
|
176
|
+
};
|
|
177
|
+
|
|
178
|
+
const isHandlerLike = (value: unknown): value is Toolkit.WithHandler<Record<string, Tool.Any>> => {
|
|
179
|
+
return typeof (value as any).tools === 'object' && typeof (value as any).handle === 'function';
|
|
180
|
+
};
|