@dxos/assistant 0.8.2-main.2f9c567 → 0.8.2-main.30e4dbb
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/lib/browser/{chunk-OLO6SVOL.mjs → chunk-YSYP7S4W.mjs} +693 -229
- package/dist/lib/browser/chunk-YSYP7S4W.mjs.map +7 -0
- package/dist/lib/browser/index.mjs +4 -443
- package/dist/lib/browser/index.mjs.map +4 -4
- package/dist/lib/browser/meta.json +1 -1
- package/dist/lib/browser/testing/index.mjs +42 -1
- package/dist/lib/browser/testing/index.mjs.map +4 -4
- package/dist/lib/node/{chunk-CI5Q7FBG.cjs → chunk-FTIDW4BA.cjs} +714 -257
- package/dist/lib/node/chunk-FTIDW4BA.cjs.map +7 -0
- package/dist/lib/node/index.cjs +34 -470
- package/dist/lib/node/index.cjs.map +4 -4
- package/dist/lib/node/meta.json +1 -1
- package/dist/lib/node/testing/index.cjs +49 -9
- package/dist/lib/node/testing/index.cjs.map +4 -4
- package/dist/lib/node-esm/{chunk-Y3SYOFFJ.mjs → chunk-GCJNPQDJ.mjs} +693 -229
- package/dist/lib/node-esm/chunk-GCJNPQDJ.mjs.map +7 -0
- package/dist/lib/node-esm/index.mjs +4 -443
- package/dist/lib/node-esm/index.mjs.map +4 -4
- package/dist/lib/node-esm/meta.json +1 -1
- package/dist/lib/node-esm/testing/index.mjs +42 -1
- package/dist/lib/node-esm/testing/index.mjs.map +4 -4
- package/dist/types/src/ai-service/parser.d.ts.map +1 -1
- package/dist/types/src/ai-service/stream.d.ts.map +1 -1
- package/dist/types/src/ai-service/testing/test-stream.d.ts.map +1 -1
- package/dist/types/src/ai-service/types.d.ts.map +1 -1
- package/dist/types/src/ai-service/util/sse.d.ts.map +1 -1
- package/dist/types/src/{conversation → experimental/conversation}/conversation.d.ts +2 -18
- package/dist/types/src/experimental/conversation/conversation.d.ts.map +1 -0
- package/dist/types/src/experimental/conversation/conversation.test.d.ts.map +1 -0
- package/dist/types/src/experimental/conversation/index.d.ts.map +1 -0
- package/dist/types/src/experimental/conversation/types.d.ts.map +1 -0
- package/dist/types/src/experimental/cypher/ast.d.ts.map +1 -0
- package/dist/types/src/experimental/cypher/combinators.d.ts.map +1 -0
- package/dist/types/src/experimental/cypher/combinators.test.d.ts.map +1 -0
- package/dist/types/src/experimental/cypher/echo-data-source.d.ts.map +1 -0
- package/dist/types/src/experimental/cypher/execution-plan.d.ts.map +1 -0
- package/dist/types/src/experimental/cypher/execution-plan.test.d.ts.map +1 -0
- package/dist/types/src/experimental/cypher/index.d.ts.map +1 -0
- package/dist/types/src/experimental/cypher/parjs.d.ts.map +1 -0
- package/dist/types/src/experimental/cypher/parser.d.ts.map +1 -0
- package/dist/types/src/experimental/cypher/query-executor.d.ts.map +1 -0
- package/dist/types/src/experimental/cypher/query-executor.test.d.ts.map +1 -0
- package/dist/types/src/experimental/cypher/schema.d.ts.map +1 -0
- package/dist/types/src/experimental/cypher/schema.test.d.ts.map +1 -0
- package/dist/types/src/experimental/index.d.ts +3 -0
- package/dist/types/src/experimental/index.d.ts.map +1 -0
- package/dist/types/src/experimental/ner/ner.test.d.ts +2 -0
- package/dist/types/src/experimental/ner/ner.test.d.ts.map +1 -0
- package/dist/types/src/experimental/research/exa.d.ts +6 -0
- package/dist/types/src/experimental/research/exa.d.ts.map +1 -0
- package/dist/types/src/experimental/research/research.test.d.ts +2 -0
- package/dist/types/src/experimental/research/research.test.d.ts.map +1 -0
- package/dist/types/src/index.d.ts +1 -2
- package/dist/types/src/index.d.ts.map +1 -1
- package/dist/types/src/session/index.d.ts +1 -0
- package/dist/types/src/session/index.d.ts.map +1 -1
- package/dist/types/src/session/session.d.ts +1 -0
- package/dist/types/src/session/session.d.ts.map +1 -1
- package/dist/types/src/session/tools.d.ts +18 -0
- package/dist/types/src/session/tools.d.ts.map +1 -0
- package/dist/types/src/session/version-pin.d.ts +5 -0
- package/dist/types/src/session/version-pin.d.ts.map +1 -1
- package/dist/types/src/testing/index.d.ts +1 -0
- package/dist/types/src/testing/index.d.ts.map +1 -1
- package/dist/types/src/testing/logger.d.ts +1 -1
- package/dist/types/src/testing/logger.d.ts.map +1 -1
- package/dist/types/src/testing/ollama.d.ts.map +1 -1
- package/dist/types/src/testing/printer.d.ts +6 -0
- package/dist/types/src/testing/printer.d.ts.map +1 -0
- package/dist/types/src/testing/query-promts.d.ts +1 -1
- package/dist/types/src/testing/query-promts.d.ts.map +1 -1
- package/dist/types/src/testing/test-data.d.ts +1 -1
- package/dist/types/src/testing/test-data.d.ts.map +1 -1
- package/dist/types/src/util/trim.d.ts.map +1 -1
- package/dist/types/tsconfig.tsbuildinfo +1 -1
- package/package.json +19 -14
- package/src/ai-service/ollama-client.ts +1 -1
- package/src/bin/chat.ts +1 -1
- package/src/{conversation → experimental/conversation}/conversation.test.ts +2 -2
- package/src/{conversation → experimental/conversation}/conversation.ts +3 -112
- package/src/{cypher → experimental/cypher}/query-executor.test.ts +1 -1
- package/src/{cypher → experimental/cypher}/schema.test.ts +1 -1
- package/src/{cypher → experimental/cypher}/schema.ts +1 -1
- package/src/experimental/index.ts +6 -0
- package/src/experimental/ner/ner.test.ts +163 -0
- package/src/experimental/research/exa.ts +38 -0
- package/src/experimental/research/instructions.tpl +64 -0
- package/src/experimental/research/research.test.ts +263 -0
- package/src/index.ts +3 -2
- package/src/session/index.ts +1 -0
- package/src/session/session.test.ts +6 -41
- package/src/session/session.ts +11 -1
- package/src/session/tools.ts +117 -0
- package/src/session/version-pin.ts +6 -0
- package/src/testing/index.ts +1 -0
- package/src/testing/logger.ts +1 -1
- package/src/testing/printer.ts +41 -0
- package/src/testing/query-promts.ts +1 -1
- package/src/testing/query.test.ts +1 -2
- package/src/testing/test-data.ts +1 -1
- package/src/typedefs.d.ts +8 -0
- package/dist/lib/browser/chunk-OLO6SVOL.mjs.map +0 -7
- package/dist/lib/node/chunk-CI5Q7FBG.cjs.map +0 -7
- package/dist/lib/node-esm/chunk-Y3SYOFFJ.mjs.map +0 -7
- package/dist/types/src/conversation/conversation.d.ts.map +0 -1
- package/dist/types/src/conversation/conversation.test.d.ts.map +0 -1
- package/dist/types/src/conversation/index.d.ts.map +0 -1
- package/dist/types/src/conversation/types.d.ts.map +0 -1
- package/dist/types/src/cypher/ast.d.ts.map +0 -1
- package/dist/types/src/cypher/combinators.d.ts.map +0 -1
- package/dist/types/src/cypher/combinators.test.d.ts.map +0 -1
- package/dist/types/src/cypher/echo-data-source.d.ts.map +0 -1
- package/dist/types/src/cypher/execution-plan.d.ts.map +0 -1
- package/dist/types/src/cypher/execution-plan.test.d.ts.map +0 -1
- package/dist/types/src/cypher/index.d.ts.map +0 -1
- package/dist/types/src/cypher/parjs.d.ts.map +0 -1
- package/dist/types/src/cypher/parser.d.ts.map +0 -1
- package/dist/types/src/cypher/query-executor.d.ts.map +0 -1
- package/dist/types/src/cypher/query-executor.test.d.ts.map +0 -1
- package/dist/types/src/cypher/schema.d.ts.map +0 -1
- package/dist/types/src/cypher/schema.test.d.ts.map +0 -1
- /package/dist/types/src/{conversation → experimental/conversation}/conversation.test.d.ts +0 -0
- /package/dist/types/src/{conversation → experimental/conversation}/index.d.ts +0 -0
- /package/dist/types/src/{conversation → experimental/conversation}/types.d.ts +0 -0
- /package/dist/types/src/{cypher → experimental/cypher}/ast.d.ts +0 -0
- /package/dist/types/src/{cypher → experimental/cypher}/combinators.d.ts +0 -0
- /package/dist/types/src/{cypher → experimental/cypher}/combinators.test.d.ts +0 -0
- /package/dist/types/src/{cypher → experimental/cypher}/echo-data-source.d.ts +0 -0
- /package/dist/types/src/{cypher → experimental/cypher}/execution-plan.d.ts +0 -0
- /package/dist/types/src/{cypher → experimental/cypher}/execution-plan.test.d.ts +0 -0
- /package/dist/types/src/{cypher → experimental/cypher}/index.d.ts +0 -0
- /package/dist/types/src/{cypher → experimental/cypher}/parjs.d.ts +0 -0
- /package/dist/types/src/{cypher → experimental/cypher}/parser.d.ts +0 -0
- /package/dist/types/src/{cypher → experimental/cypher}/query-executor.d.ts +0 -0
- /package/dist/types/src/{cypher → experimental/cypher}/query-executor.test.d.ts +0 -0
- /package/dist/types/src/{cypher → experimental/cypher}/schema.d.ts +0 -0
- /package/dist/types/src/{cypher → experimental/cypher}/schema.test.d.ts +0 -0
- /package/src/{conversation → experimental/conversation}/index.ts +0 -0
- /package/src/{conversation → experimental/conversation}/types.ts +0 -0
- /package/src/{cypher → experimental/cypher}/ast.ts +0 -0
- /package/src/{cypher → experimental/cypher}/combinators.test.ts +0 -0
- /package/src/{cypher → experimental/cypher}/combinators.ts +0 -0
- /package/src/{cypher → experimental/cypher}/echo-data-source.ts +0 -0
- /package/src/{cypher → experimental/cypher}/execution-plan.test.ts +0 -0
- /package/src/{cypher → experimental/cypher}/execution-plan.ts +0 -0
- /package/src/{cypher → experimental/cypher}/index.ts +0 -0
- /package/src/{cypher → experimental/cypher}/parjs-shims.d.ts +0 -0
- /package/src/{cypher → experimental/cypher}/parjs.ts +0 -0
- /package/src/{cypher → experimental/cypher}/parser.ts +0 -0
- /package/src/{cypher → experimental/cypher}/query-executor.ts +0 -0
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/experimental/index.ts"],"names":[],"mappings":"AAIA,cAAc,gBAAgB,CAAC;AAC/B,cAAc,UAAU,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ner.test.d.ts","sourceRoot":"","sources":["../../../../../src/experimental/ner/ner.test.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"exa.d.ts","sourceRoot":"","sources":["../../../../../src/experimental/research/exa.ts"],"names":[],"mappings":"AAUA,KAAK,oBAAoB,GAAG;IAC1B,MAAM,EAAE,MAAM,CAAC;CAChB,CAAC;AAEF,eAAO,MAAM,aAAa,GAAI,YAAY,oBAAoB,kCAuB7D,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"research.test.d.ts","sourceRoot":"","sources":["../../../../../src/experimental/research/research.test.ts"],"names":[],"mappings":""}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/index.ts"],"names":[],"mappings":"AAIA,cAAc,cAAc,CAAC;AAC7B,cAAc,
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/index.ts"],"names":[],"mappings":"AAIA,cAAc,cAAc,CAAC;AAC7B,cAAc,WAAW,CAAC;AAG1B,cAAc,gBAAgB,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/session/index.ts"],"names":[],"mappings":"AAIA,cAAc,WAAW,CAAC;AAC1B,cAAc,eAAe,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/session/index.ts"],"names":[],"mappings":"AAIA,cAAc,WAAW,CAAC;AAC1B,cAAc,eAAe,CAAC;AAC9B,cAAc,SAAS,CAAC"}
|
|
@@ -220,6 +220,7 @@ export declare class AISession {
|
|
|
220
220
|
}>;
|
|
221
221
|
constructor(_options: AiSessionOptions);
|
|
222
222
|
run(options: SessionRunOptions): Promise<Message[]>;
|
|
223
|
+
runStructured<S extends Schema.Schema.AnyNoContext>(schema: S, options: SessionRunOptions): Promise<Schema.Schema.Type<S>>;
|
|
223
224
|
private _formatUserPrompt;
|
|
224
225
|
private _createQueryArtifactsTool;
|
|
225
226
|
private _createRequireTool;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"session.d.ts","sourceRoot":"","sources":["../../../../src/session/session.ts"],"names":[],"mappings":"AAIA,OAAO,EAAU,MAAM,EAAE,MAAM,QAAQ,CAAC;AAExC,OAAO,EAEL,OAAO,
|
|
1
|
+
{"version":3,"file":"session.d.ts","sourceRoot":"","sources":["../../../../src/session/session.ts"],"names":[],"mappings":"AAIA,OAAO,EAAU,MAAM,EAAE,MAAM,QAAQ,CAAC;AAExC,OAAO,EAEL,OAAO,EAGP,KAAK,kBAAkB,EAEvB,KAAK,IAAI,EACV,MAAM,gBAAgB,CAAC;AACxB,OAAO,EAAE,KAAK,EAAgB,MAAM,aAAa,CAAC;AAClD,OAAO,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAC9C,OAAO,EAAU,KAAK,QAAQ,EAAE,MAAM,mBAAmB,CAAC;AAM1D,OAAO,EAAqB,KAAK,eAAe,EAAE,KAAK,eAAe,EAAyB,MAAM,eAAe,CAAC;AAErH;;;;;;;;;GASG;AAEH;;;;;GAKG;AACH,MAAM,MAAM,oBAAoB,GAAG,CAAC,SAAS,EAAE;IAAE,EAAE,EAAE,QAAQ,CAAC;IAAC,WAAW,EAAE,aAAa,CAAA;CAAE,EAAE,KAAK,OAAO,CACvG,GAAG,CACD,QAAQ,EACR;IACE,OAAO,EAAE,aAAa,CAAC;IACvB,IAAI,CAAC,EAAE,MAAM,CAAC;CACf,CACF,CACF,CAAC;AAEF,MAAM,MAAM,iBAAiB,GAAG;IAC9B,MAAM,EAAE,eAAe,CAAC;IAExB,SAAS,EAAE,kBAAkB,EAAE,CAAC;IAEhC;;OAEG;IACH,KAAK,EAAE,IAAI,EAAE,CAAC;IAEd,OAAO,EAAE,OAAO,EAAE,CAAC;IAEnB,MAAM,EAAE,MAAM,CAAC;IAEf,iBAAiB,CAAC,EAAE,IAAI,CAAC,eAAe,EAAE,OAAO,CAAC,CAAC;IAEnD,UAAU,CAAC,EAAE,qBAAqB,CAAC;IAEnC,YAAY,CAAC,EAAE,MAAM,CAAC;IAEtB;;OAEG;IACH,mBAAmB,CAAC,EAAE,MAAM,EAAE,CAAC;IAE/B;;OAEG;IACH,oBAAoB,CAAC,EAAE,oBAAoB,CAAC;CAC7C,CAAC;AAEF,KAAK,cAAc,GAAG,UAAU,GAAG,WAAW,GAAG,YAAY,CAAC;AAE9D,MAAM,MAAM,gBAAgB,GAAG;IAC7B;;;;;;OAMG;IACH,cAAc,EAAE,cAAc,CAAC;CAChC,CAAC;AAEF,qBAAa,SAAS;IAmCR,OAAO,CAAC,QAAQ,CAAC,QAAQ;IAlCrC,yBAAyB;IACzB,OAAO,CAAC,QAAQ,CAAC,OAAO,CAA2B;IAEnD,yDAAyD;IACzD,OAAO,CAAC,QAAQ,CAAiB;IAEjC,kCAAkC;IAClC,OAAO,CAAC,OAAO,CAA+B;IAE9C,gCAAgC;IAChC,OAAO,CAAC,QAAQ,CAAiB;IAEjC;;OAEG;IACH,SAAgB,OAAO;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAAwB;IAE/C;;OAEG;IACH,SAAgB,KAAK;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAAsB;IAE3C;;OAEG;IACH,SAAgB,MAAM;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAAuB;IAE7C,SAAgB,WAAW,4CAA4B;IAEvD;;OAEG;IACH,SAAgB,WAAW;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAAwB;gBAEtB,QAAQ,EAAE,gBAAgB;IAQjD,GAAG,CAAC,OAAO,EAAE,iBAAiB,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC;IA6GnD,aAAa,CAAC,CAAC,SAAS,MAAM,CAAC,MAAM,CAAC,YAAY,EAAE,MAAM,EAAE,CAAC,EAAE,OAAO,EAAE,iBAAiB;YASjF,iBAAiB;IAsC/B,OAAO,CAAC,yBAAyB;IAiBjC,OAAO,CAAC,kBAAkB;IA+B1B,OAAO,CAAC,mBAAmB;IAoF3B,KAAK;CAGN;;;;AAuDD,qBAAa,wBAAyB,SAAQ,6BAG7C;CAAG"}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import { Message, type Tool } from '@dxos/artifact';
|
|
2
|
+
export declare const isToolUse: (message: Message, { onlyToolNames }?: {
|
|
3
|
+
onlyToolNames?: string[];
|
|
4
|
+
}) => boolean;
|
|
5
|
+
export type RunToolsOptions = {
|
|
6
|
+
message: Message;
|
|
7
|
+
tools: Tool[];
|
|
8
|
+
extensions?: ToolContextExtensions;
|
|
9
|
+
};
|
|
10
|
+
export type RunToolsResult = {
|
|
11
|
+
type: 'continue';
|
|
12
|
+
message: Message;
|
|
13
|
+
} | {
|
|
14
|
+
type: 'break';
|
|
15
|
+
result: unknown;
|
|
16
|
+
};
|
|
17
|
+
export declare const runTools: ({ message, tools, extensions }: RunToolsOptions) => Promise<RunToolsResult>;
|
|
18
|
+
//# sourceMappingURL=tools.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"tools.d.ts","sourceRoot":"","sources":["../../../../src/session/tools.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,OAAO,EAAE,KAAK,IAAI,EAAc,MAAM,gBAAgB,CAAC;AAKhE,eAAO,MAAM,SAAS,GAAI,SAAS,OAAO,EAAE,oBAAmB;IAAE,aAAa,CAAC,EAAE,MAAM,EAAE,CAAA;CAAO,YAI/F,CAAC;AAEF,MAAM,MAAM,eAAe,GAAG;IAC5B,OAAO,EAAE,OAAO,CAAC;IACjB,KAAK,EAAE,IAAI,EAAE,CAAC;IACd,UAAU,CAAC,EAAE,qBAAqB,CAAC;CACpC,CAAC;AAEF,MAAM,MAAM,cAAc,GACtB;IACE,IAAI,EAAE,UAAU,CAAC;IACjB,OAAO,EAAE,OAAO,CAAC;CAClB,GACD;IACE,IAAI,EAAE,OAAO,CAAC;IACd,MAAM,EAAE,OAAO,CAAC;CACjB,CAAC;AAEN,eAAO,MAAM,QAAQ,GAAU,gCAAgC,eAAe,KAAG,OAAO,CAAC,cAAc,CAqFtG,CAAC"}
|
|
@@ -6,6 +6,11 @@ declare const VersionPinSchema: Schema.Struct<{
|
|
|
6
6
|
objectId: import("@dxos/echo-schema").ObjectIdClass;
|
|
7
7
|
version: Schema.Schema<ObjectVersion, ObjectVersion, never>;
|
|
8
8
|
}>;
|
|
9
|
+
/**
|
|
10
|
+
* Used to pin a specific version of an object during the course of a conversation.
|
|
11
|
+
*
|
|
12
|
+
* This allows us to detect when the object changes during the course of the conversation.
|
|
13
|
+
*/
|
|
9
14
|
export interface VersionPin extends Schema.Schema.Type<typeof VersionPinSchema> {
|
|
10
15
|
}
|
|
11
16
|
export declare const VersionPin: typeof VersionPinSchema & {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"version-pin.d.ts","sourceRoot":"","sources":["../../../../src/session/version-pin.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAEhC,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,gBAAgB,CAAC;AAC1D,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAEnD,OAAO,EAAY,KAAK,cAAc,EAAE,MAAM,mBAAmB,CAAC;AAKlE,QAAA,MAAM,gBAAgB;;;EAKpB,CAAC;AACH,MAAM,WAAW,UAAW,SAAQ,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,gBAAgB,CAAC;CAAG;AAElF,eAAO,MAAM,UAAU,EAAE,OAAO,gBAAgB,GAAG;IACjD,WAAW,EAAE,aAAa,CAAC;IAC3B,UAAU,EAAE,CAAC,MAAM,EAAE,cAAc,KAAK,UAAU,CAAC;IACnD,WAAW,EAAE,CAAC,GAAG,EAAE,UAAU,KAAK,mBAAmB,CAAC;CAiBvD,CAAC"}
|
|
1
|
+
{"version":3,"file":"version-pin.d.ts","sourceRoot":"","sources":["../../../../src/session/version-pin.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAEhC,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,gBAAgB,CAAC;AAC1D,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAEnD,OAAO,EAAY,KAAK,cAAc,EAAE,MAAM,mBAAmB,CAAC;AAKlE,QAAA,MAAM,gBAAgB;;;EAKpB,CAAC;AAEH;;;;GAIG;AACH,MAAM,WAAW,UAAW,SAAQ,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,gBAAgB,CAAC;CAAG;AAElF,eAAO,MAAM,UAAU,EAAE,OAAO,gBAAgB,GAAG;IACjD,WAAW,EAAE,aAAa,CAAC;IAC3B,UAAU,EAAE,CAAC,MAAM,EAAE,cAAc,KAAK,UAAU,CAAC;IACnD,WAAW,EAAE,CAAC,GAAG,EAAE,UAAU,KAAK,mBAAmB,CAAC;CAiBvD,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/testing/index.ts"],"names":[],"mappings":"AAIA,cAAc,QAAQ,CAAC;AACvB,cAAc,UAAU,CAAC;AACzB,cAAc,UAAU,CAAC;AACzB,cAAc,gBAAgB,CAAC;AAC/B,cAAc,aAAa,CAAC;AAC5B,cAAc,eAAe,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/testing/index.ts"],"names":[],"mappings":"AAIA,cAAc,QAAQ,CAAC;AACvB,cAAc,UAAU,CAAC;AACzB,cAAc,UAAU,CAAC;AACzB,cAAc,gBAAgB,CAAC;AAC/B,cAAc,aAAa,CAAC;AAC5B,cAAc,eAAe,CAAC;AAC9B,cAAc,WAAW,CAAC"}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { ImageSource } from '@dxos/artifact';
|
|
2
|
-
import type { ConversationEvent } from '../
|
|
2
|
+
import type { ConversationEvent } from '../experimental';
|
|
3
3
|
export declare const createLogger: ({ stream, filter, onImage, }?: {
|
|
4
4
|
stream?: boolean;
|
|
5
5
|
filter?: (event: ConversationEvent) => boolean;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"logger.d.ts","sourceRoot":"","sources":["../../../../src/testing/logger.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,WAAW,EAAuB,MAAM,gBAAgB,CAAC;AAEvE,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AAEzD,eAAO,MAAM,YAAY,
|
|
1
|
+
{"version":3,"file":"logger.d.ts","sourceRoot":"","sources":["../../../../src/testing/logger.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,WAAW,EAAuB,MAAM,gBAAgB,CAAC;AAEvE,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AAEzD,eAAO,MAAM,YAAY,GAAI,+BAI1B;IACD,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,MAAM,CAAC,EAAE,CAAC,KAAK,EAAE,iBAAiB,KAAK,OAAO,CAAC;IAC/C,OAAO,CAAC,EAAE,CAAC,GAAG,EAAE;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,WAAW,CAAC,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,WAAW,CAAA;KAAE,KAAK,IAAI,CAAC;CAC/E,MAII,OAAO,iBAAiB,SA4GjC,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ollama.d.ts","sourceRoot":"","sources":["../../../../src/testing/ollama.ts"],"names":[],"mappings":"AAIA,OAAO,EAAwB,YAAY,EAAE,KAAK,kBAAkB,EAAE,MAAM,eAAe,CAAC;AAE5F;;GAEG;AACH,eAAO,MAAM,sBAAsB,
|
|
1
|
+
{"version":3,"file":"ollama.d.ts","sourceRoot":"","sources":["../../../../src/testing/ollama.ts"],"names":[],"mappings":"AAIA,OAAO,EAAwB,YAAY,EAAE,KAAK,kBAAkB,EAAE,MAAM,eAAe,CAAC;AAE5F;;GAEG;AACH,eAAO,MAAM,sBAAsB,GAAI,UAAU,IAAI,CAAC,kBAAkB,EAAE,OAAO,CAAC,iBAKjF,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"printer.d.ts","sourceRoot":"","sources":["../../../../src/testing/printer.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,EAAE,OAAO,EAAE,mBAAmB,EAAE,MAAM,gBAAgB,CAAC;AAEnE,qBAAa,cAAc;IACzB,YAAY,GAAI,SAAS,OAAO,UAM9B;IAEF,iBAAiB,GAAI,SAAS,mBAAmB,UAsB/C;CACH"}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { Schema } from 'effect';
|
|
2
|
-
import { type DataSource } from '../
|
|
2
|
+
import { type DataSource } from '../experimental';
|
|
3
3
|
export declare const createCypherTool: (dataSource: DataSource, schemaTypes?: Schema.Schema.Any[]) => import("@dxos/artifact").Tool;
|
|
4
4
|
export declare const createSystemPrompt: (schemaTypes: Schema.Schema.Any[]) => string;
|
|
5
5
|
//# sourceMappingURL=query-promts.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"query-promts.d.ts","sourceRoot":"","sources":["../../../../src/testing/query-promts.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAMhC,OAAO,EAAwC,KAAK,UAAU,EAAE,MAAM,
|
|
1
|
+
{"version":3,"file":"query-promts.d.ts","sourceRoot":"","sources":["../../../../src/testing/query-promts.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAMhC,OAAO,EAAwC,KAAK,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAGxF,eAAO,MAAM,gBAAgB,GAAI,YAAY,UAAU,EAAE,cAAa,MAAM,CAAC,MAAM,CAAC,GAAG,EAAO,kCA0C1F,CAAC;AAEL,eAAO,MAAM,kBAAkB,GAAI,aAAa,MAAM,CAAC,MAAM,CAAC,GAAG,EAAE,WAiBlE,CAAC"}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { type Schema } from 'effect';
|
|
2
2
|
import type { EchoDatabase } from '@dxos/echo-db';
|
|
3
3
|
import { type JsonSchemaType } from '@dxos/echo-schema';
|
|
4
|
-
import { type DataSource, type Node, type Relationship } from '../
|
|
4
|
+
import { type DataSource, type Node, type Relationship } from '../experimental';
|
|
5
5
|
export declare const seedTestData: (db: EchoDatabase) => void;
|
|
6
6
|
export declare const createTestData: () => MockDataSource;
|
|
7
7
|
export declare class MockDataSource implements DataSource {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"test-data.d.ts","sourceRoot":"","sources":["../../../../src/testing/test-data.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,KAAK,MAAM,EAAE,MAAM,QAAQ,CAAC;AAGrC,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAClD,OAAO,EAIL,KAAK,cAAc,EAGpB,MAAM,mBAAmB,CAAC;AAI3B,OAAO,EACL,KAAK,UAAU,EACf,KAAK,IAAI,EACT,KAAK,YAAY,EAGlB,MAAM,
|
|
1
|
+
{"version":3,"file":"test-data.d.ts","sourceRoot":"","sources":["../../../../src/testing/test-data.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,KAAK,MAAM,EAAE,MAAM,QAAQ,CAAC;AAGrC,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAClD,OAAO,EAIL,KAAK,cAAc,EAGpB,MAAM,mBAAmB,CAAC;AAI3B,OAAO,EACL,KAAK,UAAU,EACf,KAAK,IAAI,EACT,KAAK,YAAY,EAGlB,MAAM,iBAAiB,CAAC;AAGzB,eAAO,MAAM,YAAY,GAAI,IAAI,YAAY,SAoH5C,CAAC;AAGF,eAAO,MAAM,cAAc,QAAO,cAuGjC,CAAC;AAEF,qBAAa,cAAe,YAAW,UAAU;IAC/C,OAAO,EAAE,MAAM,CACb,MAAM,EACN;QACE,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC;QAC1B,OAAO,EAAE,MAAM,CAAC;QAChB,EAAE,EAAE,MAAM,CAAC;QACX,IAAI,EAAE,OAAO,CAAC;KACf,CACF,CAAM;IAEP,MAAM,EAAE,MAAM,CACZ,MAAM,EACN;QACE,IAAI,EAAE,cAAc,CAAC;QACrB,QAAQ,EAAE,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC;KAC7B,CACF,CAAM;IAEP,KAAK,EAAE,IAAI,EAAE,CAAM;IACnB,aAAa,EAAE,YAAY,EAAE,CAAM;IAE7B,QAAQ,CAAC,EAAE,KAAK,EAAE,EAAE;QAAE,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE;IAItC,gBAAgB,CAAC,EAAE,KAAK,EAAE,EAAE;QAAE,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE;IAIpD,GAAG,CAAC,CAAC,SAAS,MAAM,CAAC,MAAM,CAAC,GAAG,EAAE,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;IAsBvE,YAAY;CA0Eb"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"trim.d.ts","sourceRoot":"","sources":["../../../../src/util/trim.ts"],"names":[],"mappings":"AAIA;;GAEG;AAEH,eAAO,MAAM,IAAI,
|
|
1
|
+
{"version":3,"file":"trim.d.ts","sourceRoot":"","sources":["../../../../src/util/trim.ts"],"names":[],"mappings":"AAIA;;GAEG;AAEH,eAAO,MAAM,IAAI,GAAI,SAAS,oBAAoB,EAAE,GAAG,QAAQ,GAAG,EAAE,WAMnE,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":"5.
|
|
1
|
+
{"version":"5.8.3"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@dxos/assistant",
|
|
3
|
-
"version": "0.8.2-main.
|
|
3
|
+
"version": "0.8.2-main.30e4dbb",
|
|
4
4
|
"description": "Natural language interface to DXOS",
|
|
5
5
|
"homepage": "https://dxos.org",
|
|
6
6
|
"bugs": "https://github.com/dxos/dxos/issues",
|
|
@@ -37,22 +37,27 @@
|
|
|
37
37
|
"src"
|
|
38
38
|
],
|
|
39
39
|
"dependencies": {
|
|
40
|
+
"@xenova/transformers": "^2.17.2",
|
|
41
|
+
"exa-js": "^1.5.12",
|
|
40
42
|
"parjs": "^1.3.9",
|
|
41
43
|
"parsimmon": "^1.18.1",
|
|
42
44
|
"ulidx": "^2.3.0",
|
|
43
|
-
"@dxos/artifact": "0.8.2-main.
|
|
44
|
-
"@dxos/
|
|
45
|
-
"@dxos/
|
|
46
|
-
"@dxos/debug": "0.8.2-main.
|
|
47
|
-
"@dxos/echo": "0.8.2-main.
|
|
48
|
-
"@dxos/echo-
|
|
49
|
-
"@dxos/echo
|
|
50
|
-
"@dxos/
|
|
51
|
-
"@dxos/
|
|
52
|
-
"@dxos/
|
|
53
|
-
"@dxos/
|
|
54
|
-
"@dxos/
|
|
55
|
-
"@dxos/
|
|
45
|
+
"@dxos/artifact": "0.8.2-main.30e4dbb",
|
|
46
|
+
"@dxos/async": "0.8.2-main.30e4dbb",
|
|
47
|
+
"@dxos/context": "0.8.2-main.30e4dbb",
|
|
48
|
+
"@dxos/debug": "0.8.2-main.30e4dbb",
|
|
49
|
+
"@dxos/echo-db": "0.8.2-main.30e4dbb",
|
|
50
|
+
"@dxos/echo-protocol": "0.8.2-main.30e4dbb",
|
|
51
|
+
"@dxos/echo": "0.8.2-main.30e4dbb",
|
|
52
|
+
"@dxos/echo-schema": "0.8.2-main.30e4dbb",
|
|
53
|
+
"@dxos/invariant": "0.8.2-main.30e4dbb",
|
|
54
|
+
"@dxos/effect": "0.8.2-main.30e4dbb",
|
|
55
|
+
"@dxos/live-object": "0.8.2-main.30e4dbb",
|
|
56
|
+
"@dxos/keys": "0.8.2-main.30e4dbb",
|
|
57
|
+
"@dxos/node-std": "0.8.2-main.30e4dbb",
|
|
58
|
+
"@dxos/log": "0.8.2-main.30e4dbb",
|
|
59
|
+
"@dxos/schema": "0.8.2-main.30e4dbb",
|
|
60
|
+
"@dxos/util": "0.8.2-main.30e4dbb"
|
|
56
61
|
},
|
|
57
62
|
"devDependencies": {
|
|
58
63
|
"@types/parsimmon": "^1.10.9",
|
|
@@ -14,7 +14,7 @@ import { MessageCollector, emitMessageAsEvents } from './message-collector';
|
|
|
14
14
|
import { type AIServiceClient, type GenerationStream } from './service';
|
|
15
15
|
import { GenerationStreamImpl } from './stream';
|
|
16
16
|
import { ToolTypes, type GenerateRequest, type GenerateResponse, type GenerationStreamEvent } from './types';
|
|
17
|
-
import { isToolUse, runTools } from '../
|
|
17
|
+
import { isToolUse, runTools } from '../session';
|
|
18
18
|
|
|
19
19
|
export type OllamaClientParams = {
|
|
20
20
|
endpoint?: string;
|
package/src/bin/chat.ts
CHANGED
|
@@ -11,7 +11,7 @@ import { SpaceId } from '@dxos/keys';
|
|
|
11
11
|
import { log } from '@dxos/log';
|
|
12
12
|
|
|
13
13
|
import { AIServiceEdgeClient, ToolTypes, DEFAULT_EDGE_MODEL } from '../ai-service';
|
|
14
|
-
import { runLLM } from '../
|
|
14
|
+
import { runLLM } from '../experimental';
|
|
15
15
|
import {
|
|
16
16
|
AI_SERVICE_ENDPOINT,
|
|
17
17
|
createLogger,
|
|
@@ -11,8 +11,8 @@ import { SpaceId } from '@dxos/keys';
|
|
|
11
11
|
import { log } from '@dxos/log';
|
|
12
12
|
|
|
13
13
|
import { runLLM, type ConversationEvent } from './conversation';
|
|
14
|
-
import { AIServiceEdgeClient, DEFAULT_EDGE_MODEL } from '
|
|
15
|
-
import { AI_SERVICE_ENDPOINT } from '
|
|
14
|
+
import { AIServiceEdgeClient, DEFAULT_EDGE_MODEL } from '../../ai-service';
|
|
15
|
+
import { AI_SERVICE_ENDPOINT } from '../../testing';
|
|
16
16
|
|
|
17
17
|
// TODO(burdon): Local live LLM test.
|
|
18
18
|
describe.skip('Conversation tests', () => {
|
|
@@ -2,13 +2,13 @@
|
|
|
2
2
|
// Copyright 2024 DXOS.org
|
|
3
3
|
//
|
|
4
4
|
|
|
5
|
-
import { type
|
|
6
|
-
import {
|
|
5
|
+
import { type Message, type Tool } from '@dxos/artifact';
|
|
6
|
+
import { ObjectId } from '@dxos/echo-schema';
|
|
7
7
|
import { invariant } from '@dxos/invariant';
|
|
8
8
|
import { type SpaceId } from '@dxos/keys';
|
|
9
9
|
import { log } from '@dxos/log';
|
|
10
10
|
|
|
11
|
-
import { type AIServiceClient, type GenerationStreamEvent, type LLMModel } from '
|
|
11
|
+
import { type AIServiceClient, type GenerationStreamEvent, type LLMModel } from '../../ai-service';
|
|
12
12
|
|
|
13
13
|
export type CreateLLMConversationParams = {
|
|
14
14
|
model: LLMModel;
|
|
@@ -140,112 +140,3 @@ export const runLLM = async (params: CreateLLMConversationParams) => {
|
|
|
140
140
|
history,
|
|
141
141
|
};
|
|
142
142
|
};
|
|
143
|
-
|
|
144
|
-
export const isToolUse = (message: Message, { onlyToolNames }: { onlyToolNames?: string[] } = {}) => {
|
|
145
|
-
const block = message.content.at(-1);
|
|
146
|
-
invariant(block);
|
|
147
|
-
return block.type === 'tool_use' && (!onlyToolNames || onlyToolNames.includes(block.name));
|
|
148
|
-
};
|
|
149
|
-
|
|
150
|
-
export type RunToolsOptions = {
|
|
151
|
-
message: Message;
|
|
152
|
-
tools: Tool[];
|
|
153
|
-
extensions?: ToolContextExtensions;
|
|
154
|
-
};
|
|
155
|
-
|
|
156
|
-
export type RunToolsResult =
|
|
157
|
-
| {
|
|
158
|
-
type: 'continue';
|
|
159
|
-
message: Message;
|
|
160
|
-
}
|
|
161
|
-
| {
|
|
162
|
-
type: 'break';
|
|
163
|
-
result: unknown;
|
|
164
|
-
};
|
|
165
|
-
|
|
166
|
-
export const runTools = async ({ message, tools, extensions }: RunToolsOptions): Promise<RunToolsResult> => {
|
|
167
|
-
const toolCalls = message.content.filter((block) => block.type === 'tool_use');
|
|
168
|
-
invariant(toolCalls.length === 1);
|
|
169
|
-
const toolCall = toolCalls[0];
|
|
170
|
-
const tool = tools.find((tool) => tool.name === toolCall.name);
|
|
171
|
-
if (!tool) {
|
|
172
|
-
const resultMessage = create(Message, {
|
|
173
|
-
role: 'user',
|
|
174
|
-
content: [
|
|
175
|
-
{
|
|
176
|
-
type: 'tool_result',
|
|
177
|
-
toolUseId: toolCall.id,
|
|
178
|
-
content: `Tool not found: ${toolCall.name}`,
|
|
179
|
-
isError: true,
|
|
180
|
-
},
|
|
181
|
-
],
|
|
182
|
-
});
|
|
183
|
-
|
|
184
|
-
// TODO(burdon): Retry count?
|
|
185
|
-
return {
|
|
186
|
-
type: 'continue',
|
|
187
|
-
message: resultMessage,
|
|
188
|
-
};
|
|
189
|
-
}
|
|
190
|
-
|
|
191
|
-
let toolResult: ToolResult;
|
|
192
|
-
try {
|
|
193
|
-
invariant(tool.execute);
|
|
194
|
-
toolResult = await tool.execute(toolCall.input, { extensions });
|
|
195
|
-
} catch (error: any) {
|
|
196
|
-
log('tool error', { error });
|
|
197
|
-
toolResult = ToolResult.Error(error.message);
|
|
198
|
-
}
|
|
199
|
-
|
|
200
|
-
switch (toolResult.kind) {
|
|
201
|
-
case 'error': {
|
|
202
|
-
log('tool error', { message: toolResult.message });
|
|
203
|
-
const resultMessage = create(Message, {
|
|
204
|
-
role: 'user',
|
|
205
|
-
content: [
|
|
206
|
-
{
|
|
207
|
-
type: 'tool_result',
|
|
208
|
-
toolUseId: toolCall.id,
|
|
209
|
-
content: toolResult.message,
|
|
210
|
-
isError: true,
|
|
211
|
-
},
|
|
212
|
-
],
|
|
213
|
-
});
|
|
214
|
-
|
|
215
|
-
// TODO(burdon): Retry count?
|
|
216
|
-
return {
|
|
217
|
-
type: 'continue',
|
|
218
|
-
message: resultMessage,
|
|
219
|
-
};
|
|
220
|
-
}
|
|
221
|
-
|
|
222
|
-
case 'success': {
|
|
223
|
-
log('tool success', { result: toolResult.result });
|
|
224
|
-
const resultMessage = create(Message, {
|
|
225
|
-
role: 'user',
|
|
226
|
-
content: [
|
|
227
|
-
{
|
|
228
|
-
type: 'tool_result',
|
|
229
|
-
toolUseId: toolCall.id,
|
|
230
|
-
content:
|
|
231
|
-
typeof toolResult.result === 'string' ? toolResult.result : JSON.stringify(toolResult.result) ?? '',
|
|
232
|
-
},
|
|
233
|
-
...(toolResult.extractContentBlocks ?? []),
|
|
234
|
-
],
|
|
235
|
-
});
|
|
236
|
-
|
|
237
|
-
return {
|
|
238
|
-
type: 'continue',
|
|
239
|
-
message: resultMessage,
|
|
240
|
-
};
|
|
241
|
-
}
|
|
242
|
-
|
|
243
|
-
case 'break': {
|
|
244
|
-
log('tool break', { result: toolResult.result });
|
|
245
|
-
return {
|
|
246
|
-
type: 'break',
|
|
247
|
-
result: toolResult.result,
|
|
248
|
-
};
|
|
249
|
-
}
|
|
250
|
-
}
|
|
251
|
-
};
|
|
@@ -7,7 +7,7 @@ import { test } from 'vitest';
|
|
|
7
7
|
import { log } from '@dxos/log';
|
|
8
8
|
|
|
9
9
|
import { executeQuery } from './query-executor';
|
|
10
|
-
import { createTestData } from '
|
|
10
|
+
import { createTestData } from '../../testing';
|
|
11
11
|
|
|
12
12
|
test.skip('run', async ({ expect }) => {
|
|
13
13
|
const dataSource = createTestData();
|
|
@@ -7,7 +7,7 @@ import { test } from 'vitest';
|
|
|
7
7
|
import { toJsonSchema } from '@dxos/echo-schema';
|
|
8
8
|
|
|
9
9
|
import { formatJsonSchemaForLLM } from './schema';
|
|
10
|
-
import { Contact, Organization, Project, Task } from '
|
|
10
|
+
import { Contact, Organization, Project, Task } from '../../testing';
|
|
11
11
|
|
|
12
12
|
test('Organization-Project-Task-Contact schema', async ({ expect }) => {
|
|
13
13
|
const jsonSchema = [Organization, Project, Task, Contact].map((schema) => toJsonSchema(schema));
|
|
@@ -0,0 +1,163 @@
|
|
|
1
|
+
//
|
|
2
|
+
// Copyright 2025 DXOS.org
|
|
3
|
+
//
|
|
4
|
+
|
|
5
|
+
import { pipeline } from '@xenova/transformers';
|
|
6
|
+
import { beforeAll, describe, expect, test } from 'vitest';
|
|
7
|
+
|
|
8
|
+
import { log } from '@dxos/log';
|
|
9
|
+
import { createTestData } from '@dxos/schema/testing';
|
|
10
|
+
|
|
11
|
+
type NerToken = {
|
|
12
|
+
entity: string;
|
|
13
|
+
score: number;
|
|
14
|
+
index: number;
|
|
15
|
+
word: string;
|
|
16
|
+
start: number | null;
|
|
17
|
+
end: number | null;
|
|
18
|
+
};
|
|
19
|
+
|
|
20
|
+
const createTokenGroups = (tokens: NerToken[]): NerToken[][] => {
|
|
21
|
+
const tokenGroups: NerToken[][] = [];
|
|
22
|
+
|
|
23
|
+
const canBeCombined = (prevToken: NerToken, currentToken: NerToken) => {
|
|
24
|
+
const [currentState, currentEntity] = currentToken.entity.split('-'); // B, I, O
|
|
25
|
+
const [prevState, prevEntity] = prevToken.entity.split('-'); // B, I, O
|
|
26
|
+
|
|
27
|
+
if (prevEntity !== currentEntity) {
|
|
28
|
+
return false;
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
if (prevState === 'O') {
|
|
32
|
+
return false;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
if (currentState === 'I') {
|
|
36
|
+
return true;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
if ((prevState === 'B' || prevState === 'I') && currentState === 'B') {
|
|
40
|
+
return currentToken.word.startsWith('#');
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
return false;
|
|
44
|
+
};
|
|
45
|
+
|
|
46
|
+
for (const token of tokens) {
|
|
47
|
+
if (tokenGroups.length === 0) {
|
|
48
|
+
tokenGroups.push([token]);
|
|
49
|
+
continue;
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
const lastTokenGroup = tokenGroups.at(-1)!;
|
|
53
|
+
if (canBeCombined(lastTokenGroup.at(-1)!, token)) {
|
|
54
|
+
lastTokenGroup.push(token);
|
|
55
|
+
continue;
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
tokenGroups.push([token]);
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
return tokenGroups;
|
|
62
|
+
};
|
|
63
|
+
|
|
64
|
+
const combineNerTokens = (group: NerToken[]): NerToken => {
|
|
65
|
+
const combinedToken: NerToken = {
|
|
66
|
+
entity: group.at(0)!.entity.split('-')[1],
|
|
67
|
+
score: group.reduce((acc, token) => acc + token.score, 0) / group.length,
|
|
68
|
+
index: group.at(0)!.index,
|
|
69
|
+
word: group.at(0)!.word,
|
|
70
|
+
start: group.at(0)!.start,
|
|
71
|
+
end: group.at(-1)!.end,
|
|
72
|
+
};
|
|
73
|
+
|
|
74
|
+
for (const token of group.slice(1)) {
|
|
75
|
+
if (token.word.startsWith('#')) {
|
|
76
|
+
combinedToken.word += token.word.replace(/^#+/, '');
|
|
77
|
+
} else {
|
|
78
|
+
combinedToken.word += ' ' + token.word;
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
return combinedToken;
|
|
83
|
+
};
|
|
84
|
+
|
|
85
|
+
describe.skip('Named Entity Recognition', () => {
|
|
86
|
+
let ner: any;
|
|
87
|
+
|
|
88
|
+
beforeAll(async () => {
|
|
89
|
+
// Initialize the NER pipeline
|
|
90
|
+
ner = await pipeline('ner', 'Xenova/bert-base-NER');
|
|
91
|
+
});
|
|
92
|
+
|
|
93
|
+
test.skip('should identify named entities in text', async () => {
|
|
94
|
+
const text = 'Elon Musk is the CEO of SpaceX and Tesla.';
|
|
95
|
+
const result = await ner(text);
|
|
96
|
+
|
|
97
|
+
// Verify the structure of the result
|
|
98
|
+
expect(Array.isArray(result)).toBe(true);
|
|
99
|
+
expect(result.length).toBeGreaterThan(0);
|
|
100
|
+
|
|
101
|
+
log.info('ner', { result });
|
|
102
|
+
|
|
103
|
+
// Check for specific entities
|
|
104
|
+
const entities = result.map((item: any) => ({
|
|
105
|
+
text: item.word,
|
|
106
|
+
entity: item.entity,
|
|
107
|
+
score: item.score,
|
|
108
|
+
}));
|
|
109
|
+
|
|
110
|
+
// Verify Elon Musk is recognized as a person
|
|
111
|
+
const elonMusk = entities.find((e: any) => e.text === 'Elon' || e.text === 'Musk');
|
|
112
|
+
expect(elonMusk).toBeDefined();
|
|
113
|
+
expect(elonMusk?.entity).toMatch(/PER/);
|
|
114
|
+
|
|
115
|
+
// Verify SpaceX and Tesla are recognized as organizations
|
|
116
|
+
const companies = entities.filter((e: any) => e.text === 'SpaceX' || e.text === 'Tesla');
|
|
117
|
+
expect(companies.length).toBe(2);
|
|
118
|
+
companies.forEach((company: any) => {
|
|
119
|
+
expect(company.entity).toMatch(/ORG/);
|
|
120
|
+
});
|
|
121
|
+
});
|
|
122
|
+
|
|
123
|
+
test('own data', async () => {
|
|
124
|
+
const { transcriptWoflram, transcriptJosiah } = await createTestData();
|
|
125
|
+
|
|
126
|
+
const blocks = [...transcriptWoflram, ...transcriptJosiah].flatMap((message) =>
|
|
127
|
+
message.blocks.flatMap((block) => (block.type === 'transcription' ? [block.text] : [])),
|
|
128
|
+
);
|
|
129
|
+
|
|
130
|
+
for (const block of blocks) {
|
|
131
|
+
const result = await ner(block);
|
|
132
|
+
const combinedTokens = createTokenGroups(result).map(combineNerTokens);
|
|
133
|
+
log.info('ner', { text: block, result: combinedTokens });
|
|
134
|
+
}
|
|
135
|
+
});
|
|
136
|
+
|
|
137
|
+
test('more own data', async () => {
|
|
138
|
+
const { transcriptMessages } = await createTestData();
|
|
139
|
+
|
|
140
|
+
const blocks = transcriptMessages.flatMap((message) =>
|
|
141
|
+
message.blocks.flatMap((block) => (block.type === 'transcription' ? [block.text] : [])),
|
|
142
|
+
);
|
|
143
|
+
|
|
144
|
+
for (const block of blocks) {
|
|
145
|
+
const result = await ner(block);
|
|
146
|
+
const combinedTokens = createTokenGroups(result).map(combineNerTokens);
|
|
147
|
+
log.info('ner', { text: block, result: combinedTokens });
|
|
148
|
+
}
|
|
149
|
+
});
|
|
150
|
+
|
|
151
|
+
test('should handle empty text', async () => {
|
|
152
|
+
const result = await ner('');
|
|
153
|
+
expect(Array.isArray(result)).toBe(true);
|
|
154
|
+
expect(result.length).toBe(0);
|
|
155
|
+
});
|
|
156
|
+
|
|
157
|
+
test('should handle text with no named entities', async () => {
|
|
158
|
+
const text = 'The weather is nice today.';
|
|
159
|
+
const result = await ner(text);
|
|
160
|
+
expect(Array.isArray(result)).toBe(true);
|
|
161
|
+
expect(result.length).toBe(0);
|
|
162
|
+
});
|
|
163
|
+
});
|