@coasys/ad4m 0.10.0-prerelease → 0.10.0-rc10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/index.cjs +8167 -683
- package/lib/index.js +8167 -684
- package/lib/src/Ad4mClient.d.ts +2 -0
- package/lib/src/Ad4mClient.js +8 -2
- package/lib/src/PubSub.d.ts +1 -0
- package/lib/src/PubSub.js +1 -0
- package/lib/src/ai/AIClient.d.ts +19 -0
- package/lib/src/ai/AIClient.js +236 -0
- package/lib/src/ai/AIResolver.d.ts +14 -0
- package/lib/src/ai/AIResolver.js +145 -0
- package/lib/src/ai/Tasks.d.ts +37 -0
- package/lib/src/ai/Tasks.js +165 -0
- package/lib/src/buildSchema.js +3 -1
- package/lib/src/expression/ExpressionClient.d.ts +1 -1
- package/lib/src/expression/ExpressionClient.js +13 -1
- package/lib/src/index.d.ts +2 -0
- package/lib/src/index.js +2 -0
- package/lib/src/language/LanguageContext.d.ts +1 -0
- package/lib/src/perspectives/PerspectiveProxy.js +84 -60
- package/lib/src/runtime/RuntimeClient.d.ts +0 -2
- package/lib/src/runtime/RuntimeClient.js +0 -20
- package/lib/src/runtime/RuntimeResolver.d.ts +0 -1
- package/lib/src/runtime/RuntimeResolver.js +1 -21
- package/lib/src/schema.gql +48 -1
- package/lib/src/subject/SubjectEntity.js +0 -2
- package/lib/src/typeDefs.js +48 -1
- package/package.json +7 -3
- package/src/Ad4mClient.test.ts +77 -1
- package/src/Ad4mClient.ts +7 -0
- package/src/PubSub.ts +2 -1
- package/src/ai/AIClient.ts +252 -0
- package/src/ai/AIResolver.ts +147 -0
- package/src/ai/Tasks.ts +121 -0
- package/src/buildSchema.ts +3 -1
- package/src/expression/ExpressionClient.ts +14 -1
- package/src/index.ts +2 -0
- package/src/perspectives/PerspectiveProxy.ts +90 -67
- package/src/runtime/RuntimeClient.ts +0 -21
- package/src/runtime/RuntimeResolver.ts +0 -17
- package/src/subject/SubjectEntity.ts +0 -2
package/lib/src/typeDefs.js
CHANGED
|
@@ -1,5 +1,42 @@
|
|
|
1
1
|
|
|
2
2
|
export const typeDefsString = `
|
|
3
|
+
type AIModelLoadingStatus {
|
|
4
|
+
downloaded: Boolean!
|
|
5
|
+
loaded: Boolean!
|
|
6
|
+
model: String!
|
|
7
|
+
progress: Float!
|
|
8
|
+
status: String!
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
type AIPromptExamples {
|
|
12
|
+
input: String!
|
|
13
|
+
output: String!
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
input AIPromptExamplesInput {
|
|
17
|
+
input: String!
|
|
18
|
+
output: String!
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
type AITask {
|
|
22
|
+
createdAt: String!
|
|
23
|
+
metaData: String
|
|
24
|
+
modelId: String!
|
|
25
|
+
name: String!
|
|
26
|
+
promptExamples: [AIPromptExamples!]!
|
|
27
|
+
systemPrompt: String!
|
|
28
|
+
taskId: String!
|
|
29
|
+
updatedAt: String!
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
input AITaskInput {
|
|
33
|
+
metaData: String
|
|
34
|
+
modelId: String!
|
|
35
|
+
name: String!
|
|
36
|
+
promptExamples: [AIPromptExamplesInput!]!
|
|
37
|
+
systemPrompt: String!
|
|
38
|
+
}
|
|
39
|
+
|
|
3
40
|
type Agent {
|
|
4
41
|
did: String!
|
|
5
42
|
directMessageLanguage: String
|
|
@@ -225,6 +262,14 @@ type Mutation {
|
|
|
225
262
|
agentUnlock(holochain: Boolean!, passphrase: String!): AgentStatus!
|
|
226
263
|
agentUpdateDirectMessageLanguage(directMessageLanguage: String!): Agent!
|
|
227
264
|
agentUpdatePublicPerspective(perspective: PerspectiveInput!): Agent!
|
|
265
|
+
aiAddTask(task: AITaskInput!): AITask!
|
|
266
|
+
aiCloseTranscriptionStream(streamId: String!): String!
|
|
267
|
+
aiEmbed(modelId: String!, text: String!): String!
|
|
268
|
+
aiFeedTranscriptionStream(audio: [Float!]!, streamId: String!): String!
|
|
269
|
+
aiOpenTranscriptionStream(modelId: String!): String!
|
|
270
|
+
aiPrompt(prompt: String!, taskId: String!): String!
|
|
271
|
+
aiRemoveTask(taskId: String!): AITask!
|
|
272
|
+
aiUpdateTask(task: AITaskInput!, taskId: String!): AITask!
|
|
228
273
|
deleteTrustedAgents(agents: [String!]!): [String!]!
|
|
229
274
|
expressionCreate(content: String!, languageAddress: String!): String!
|
|
230
275
|
expressionInteract(interactionCall: InteractionCall!, url: String!): String
|
|
@@ -345,6 +390,8 @@ type Query {
|
|
|
345
390
|
agentGetEntanglementProofs: [EntanglementProof!]!
|
|
346
391
|
agentIsLocked: Boolean!
|
|
347
392
|
agentStatus: AgentStatus!
|
|
393
|
+
aiModelLoadingStatus(model: String!): AIModelLoadingStatus!
|
|
394
|
+
aiTasks: [AITask!]!
|
|
348
395
|
expression(url: String!): ExpressionRendered
|
|
349
396
|
expressionInteractions(url: String!): [InteractionMeta!]!
|
|
350
397
|
expressionMany(urls: [String!]!): [ExpressionRendered]!
|
|
@@ -398,6 +445,7 @@ type Subscription {
|
|
|
398
445
|
agentAppsChanged: [Apps!]
|
|
399
446
|
agentStatusChanged: AgentStatus
|
|
400
447
|
agentUpdated: Agent
|
|
448
|
+
aiTranscriptionText(streamId: String!): String!
|
|
401
449
|
exceptionOccurred: ExceptionInfo
|
|
402
450
|
neighbourhoodSignal(perspectiveUUID: String!): PerspectiveExpression
|
|
403
451
|
perspectiveAdded: PerspectiveHandle
|
|
@@ -408,7 +456,6 @@ type Subscription {
|
|
|
408
456
|
perspectiveSyncStateChange(uuid: String!): String!
|
|
409
457
|
perspectiveUpdated: PerspectiveHandle
|
|
410
458
|
runtimeMessageReceived: PerspectiveExpression
|
|
411
|
-
runtimeNotificationRequested: Notification
|
|
412
459
|
runtimeNotificationTriggered: TriggeredNotification
|
|
413
460
|
}
|
|
414
461
|
|
package/package.json
CHANGED
|
@@ -39,7 +39,9 @@
|
|
|
39
39
|
"express": "4.18.2",
|
|
40
40
|
"graphql": "15.7.2",
|
|
41
41
|
"reflect-metadata": "^0.1.13",
|
|
42
|
-
"type-graphql": "1.1.1"
|
|
42
|
+
"type-graphql": "1.1.1",
|
|
43
|
+
"pako": "*",
|
|
44
|
+
"base64-js": "*"
|
|
43
45
|
},
|
|
44
46
|
"devDependencies": {
|
|
45
47
|
"@apollo/server": "^4.9.4",
|
|
@@ -57,10 +59,12 @@
|
|
|
57
59
|
"typescript": "^4.6.2",
|
|
58
60
|
"ws": "8.13.0",
|
|
59
61
|
"@types/node": "*",
|
|
60
|
-
"@types/yargs": "*"
|
|
62
|
+
"@types/yargs": "*",
|
|
63
|
+
"react": "*",
|
|
64
|
+
"react-dom": "*"
|
|
61
65
|
},
|
|
62
66
|
"patchedDependencies": {},
|
|
63
|
-
"version": "0.10.0-
|
|
67
|
+
"version": "0.10.0-rc10",
|
|
64
68
|
"pnpm": {
|
|
65
69
|
"patchedDependencies": {
|
|
66
70
|
"graphql@15.7.2": "patches/graphql@15.7.2.patch"
|
package/src/Ad4mClient.test.ts
CHANGED
|
@@ -24,6 +24,7 @@ import NeighbourhoodResolver from "./neighbourhood/NeighbourhoodResolver";
|
|
|
24
24
|
import PerspectiveResolver from "./perspectives/PerspectiveResolver";
|
|
25
25
|
import RuntimeResolver from "./runtime/RuntimeResolver";
|
|
26
26
|
import ExpressionResolver from "./expression/ExpressionResolver";
|
|
27
|
+
import AIResolver from './ai/AIResolver'
|
|
27
28
|
import { AuthInfoInput, EntanglementProofInput, CapabilityInput, ResourceInput } from "./agent/Agent";
|
|
28
29
|
import { LanguageMetaInput } from "./language/LanguageMeta";
|
|
29
30
|
import { InteractionCall } from "./language/Language";
|
|
@@ -39,7 +40,8 @@ async function createGqlServer(port: number) {
|
|
|
39
40
|
LanguageResolver,
|
|
40
41
|
NeighbourhoodResolver,
|
|
41
42
|
PerspectiveResolver,
|
|
42
|
-
RuntimeResolver
|
|
43
|
+
RuntimeResolver,
|
|
44
|
+
AIResolver
|
|
43
45
|
]
|
|
44
46
|
})
|
|
45
47
|
|
|
@@ -1150,4 +1152,78 @@ describe('Ad4mClient', () => {
|
|
|
1150
1152
|
})
|
|
1151
1153
|
})
|
|
1152
1154
|
})
|
|
1155
|
+
describe('.ai', () => {
|
|
1156
|
+
it('embed()', async () => {
|
|
1157
|
+
const vector = await ad4mClient.ai.embed("model", "test ets")
|
|
1158
|
+
expect(vector[0]).toEqual(0)
|
|
1159
|
+
expect(vector[1]).toEqual(10)
|
|
1160
|
+
expect(vector[2]).toEqual(20)
|
|
1161
|
+
expect(vector[3]).toEqual(30)
|
|
1162
|
+
})
|
|
1163
|
+
|
|
1164
|
+
it('tasks()', async () => {
|
|
1165
|
+
const tasks = await ad4mClient.ai.tasks()
|
|
1166
|
+
expect(tasks.length).toBe(2)
|
|
1167
|
+
expect(tasks[0].taskId).toBe("task_id")
|
|
1168
|
+
expect(tasks[0].modelId).toBe("modelId")
|
|
1169
|
+
})
|
|
1170
|
+
|
|
1171
|
+
it('addTask()', async () => {
|
|
1172
|
+
const task = await ad4mClient.ai.addTask("task_name", "model_id", "system prompt", []);
|
|
1173
|
+
expect(task.name).toBe("task_name")
|
|
1174
|
+
expect(task.taskId).toBe("task_id")
|
|
1175
|
+
expect(task.modelId).toBe("model_id")
|
|
1176
|
+
expect(task.systemPrompt).toBe("system prompt")
|
|
1177
|
+
});
|
|
1178
|
+
|
|
1179
|
+
it('removeTask()', async () => {
|
|
1180
|
+
const task = await ad4mClient.ai.removeTask("task_id", "system prompt", []);
|
|
1181
|
+
expect(task.taskId).toBe("task_id")
|
|
1182
|
+
expect(task.modelId).toBe("model_id")
|
|
1183
|
+
expect(task.systemPrompt).toBe("system prompt")
|
|
1184
|
+
});
|
|
1185
|
+
|
|
1186
|
+
it('updateTask()', async () => {
|
|
1187
|
+
const task = await ad4mClient.ai.updateTask("task_id", {
|
|
1188
|
+
name: "task_name",
|
|
1189
|
+
modelId: "model_id",
|
|
1190
|
+
systemPrompt: "system prompt",
|
|
1191
|
+
promptExamples: []
|
|
1192
|
+
});
|
|
1193
|
+
expect(task.name).toBe("task_name")
|
|
1194
|
+
expect(task.taskId).toBe("task_id")
|
|
1195
|
+
expect(task.modelId).toBe("model_id")
|
|
1196
|
+
expect(task.systemPrompt).toBe("system prompt")
|
|
1197
|
+
});
|
|
1198
|
+
|
|
1199
|
+
it('modelLoadingStatus()', async () => {
|
|
1200
|
+
const status = await ad4mClient.ai.modelLoadingStatus("model_id");
|
|
1201
|
+
expect(status.status).toBe("loaded")
|
|
1202
|
+
});
|
|
1203
|
+
|
|
1204
|
+
it('prompt()', async () => {
|
|
1205
|
+
const prompt = await ad4mClient.ai.prompt("task_id", "Do something");
|
|
1206
|
+
console.log(prompt)
|
|
1207
|
+
expect(prompt).toBe("output")
|
|
1208
|
+
})
|
|
1209
|
+
|
|
1210
|
+
it('openTranscriptionStream(), closeTranscriptionStream(), feedTranscriptionStream() & aiTranscriptionText subscription', async () => {
|
|
1211
|
+
const streamCallback = jest.fn()
|
|
1212
|
+
const streamId = await ad4mClient.ai.openTranscriptionStream("model_id", streamCallback);
|
|
1213
|
+
expect(streamId).toBeTruthy()
|
|
1214
|
+
expect(streamId).toBe("streamId")
|
|
1215
|
+
expect(streamCallback).toBeCalledTimes(0)
|
|
1216
|
+
|
|
1217
|
+
await new Promise<void>(resolve => setTimeout(resolve, 100))
|
|
1218
|
+
|
|
1219
|
+
await ad4mClient.ai.feedTranscriptionStream(streamId, [0, 10, 20, 30]);
|
|
1220
|
+
|
|
1221
|
+
await new Promise<void>(resolve => setTimeout(resolve, 100))
|
|
1222
|
+
|
|
1223
|
+
expect(streamCallback).toBeCalledTimes(1)
|
|
1224
|
+
|
|
1225
|
+
const stream = await ad4mClient.ai.closeTranscriptionStream(streamId)
|
|
1226
|
+
expect(stream).toBeTruthy()
|
|
1227
|
+
})
|
|
1228
|
+
})
|
|
1153
1229
|
})
|
package/src/Ad4mClient.ts
CHANGED
|
@@ -5,6 +5,7 @@ import { NeighbourhoodClient } from './neighbourhood/NeighbourhoodClient'
|
|
|
5
5
|
import { PerspectiveClient } from './perspectives/PerspectiveClient'
|
|
6
6
|
import { RuntimeClient } from './runtime/RuntimeClient'
|
|
7
7
|
import { ExpressionClient } from './expression/ExpressionClient'
|
|
8
|
+
import { AIClient } from './ai/AIClient'
|
|
8
9
|
|
|
9
10
|
/**
|
|
10
11
|
* Client for the Ad4m interface wrapping GraphQL queryies
|
|
@@ -23,6 +24,7 @@ export class Ad4mClient {
|
|
|
23
24
|
#neighbourhoodClient: NeighbourhoodClient
|
|
24
25
|
#perspectiveClient: PerspectiveClient
|
|
25
26
|
#runtimeClient: RuntimeClient
|
|
27
|
+
#aiClient: AIClient
|
|
26
28
|
|
|
27
29
|
|
|
28
30
|
constructor(client: ApolloClient<any>, subscribe: boolean = true) {
|
|
@@ -35,6 +37,7 @@ export class Ad4mClient {
|
|
|
35
37
|
this.#perspectiveClient.setExpressionClient(this.#expressionClient)
|
|
36
38
|
this.#perspectiveClient.setNeighbourhoodClient(this.#neighbourhoodClient)
|
|
37
39
|
this.#runtimeClient = new RuntimeClient(this.#apolloClient, subscribe)
|
|
40
|
+
this.#aiClient = new AIClient(this.#apolloClient, subscribe)
|
|
38
41
|
}
|
|
39
42
|
|
|
40
43
|
get agent(): AgentClient {
|
|
@@ -60,4 +63,8 @@ export class Ad4mClient {
|
|
|
60
63
|
get runtime(): RuntimeClient {
|
|
61
64
|
return this.#runtimeClient
|
|
62
65
|
}
|
|
66
|
+
|
|
67
|
+
get ai(): AIClient {
|
|
68
|
+
return this.#aiClient
|
|
69
|
+
}
|
|
63
70
|
}
|
package/src/PubSub.ts
CHANGED
|
@@ -13,4 +13,5 @@ export const RUNTIME_NOTIFICATION_REQUESTED_TOPIC = "runtime-notification-reques
|
|
|
13
13
|
export const RUNTIME_NOTIFICATION_TRIGGERED_TOPIC = "runtime-notification-triggered-topic"
|
|
14
14
|
export const NEIGHBOURHOOD_SIGNAL_RECEIVED_TOPIC = "neighbourhood-signal-received-topic"
|
|
15
15
|
export const PERSPECTIVE_SYNC_STATE_CHANGE = "perspective-sync-state-change"
|
|
16
|
-
export const APPS_CHANGED = "apps-changed"
|
|
16
|
+
export const APPS_CHANGED = "apps-changed"
|
|
17
|
+
export const AI_TRANSCRIPTION_TEXT_TOPIC = "ai-transcription-text-topic"
|
|
@@ -0,0 +1,252 @@
|
|
|
1
|
+
import { ApolloClient, gql } from "@apollo/client";
|
|
2
|
+
import unwrapApolloResult from "../unwrapApolloResult";
|
|
3
|
+
import base64js from 'base64-js';
|
|
4
|
+
import pako from 'pako'
|
|
5
|
+
import { AIModelLoadingStatus, AITask, AITaskInput } from "./Tasks";
|
|
6
|
+
|
|
7
|
+
export class AIClient {
|
|
8
|
+
#apolloClient: ApolloClient<any>;
|
|
9
|
+
#transcriptionSubscriptions: Map<string, any> = new Map();
|
|
10
|
+
|
|
11
|
+
constructor(apolloClient: ApolloClient<any>, subscribe: boolean = true) {
|
|
12
|
+
this.#apolloClient = apolloClient;
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
async tasks(): Promise<AITask[]> {
|
|
16
|
+
const { aiTasks } = unwrapApolloResult(await this.#apolloClient.query({
|
|
17
|
+
query: gql`
|
|
18
|
+
query {
|
|
19
|
+
aiTasks {
|
|
20
|
+
name
|
|
21
|
+
modelId
|
|
22
|
+
taskId
|
|
23
|
+
systemPrompt
|
|
24
|
+
promptExamples {
|
|
25
|
+
input
|
|
26
|
+
output
|
|
27
|
+
}
|
|
28
|
+
metaData
|
|
29
|
+
createdAt
|
|
30
|
+
updatedAt
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
`
|
|
34
|
+
}));
|
|
35
|
+
|
|
36
|
+
return aiTasks;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
async addTask(name: string, modelId: string, systemPrompt: string, promptExamples: { input: string, output: string }[], metaData?: string): Promise<AITask> {
|
|
40
|
+
const task = new AITaskInput(name, modelId, systemPrompt, promptExamples, metaData);
|
|
41
|
+
const { aiAddTask } = unwrapApolloResult(await this.#apolloClient.mutate({
|
|
42
|
+
mutation: gql`
|
|
43
|
+
mutation AiAddTask($task: AITaskInput!) {
|
|
44
|
+
aiAddTask(task: $task) {
|
|
45
|
+
name
|
|
46
|
+
modelId
|
|
47
|
+
taskId
|
|
48
|
+
systemPrompt
|
|
49
|
+
promptExamples {
|
|
50
|
+
input
|
|
51
|
+
output
|
|
52
|
+
}
|
|
53
|
+
metaData
|
|
54
|
+
createdAt
|
|
55
|
+
updatedAt
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
`,
|
|
59
|
+
variables: {
|
|
60
|
+
task
|
|
61
|
+
}
|
|
62
|
+
}));
|
|
63
|
+
|
|
64
|
+
return aiAddTask;
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
async removeTask(taskId: string): Promise<AITask> {
|
|
68
|
+
const { aiRemoveTask } = unwrapApolloResult(await this.#apolloClient.mutate({
|
|
69
|
+
mutation: gql`
|
|
70
|
+
mutation AiRemoveTask($taskId: String!) {
|
|
71
|
+
aiRemoveTask(taskId: $taskId) {
|
|
72
|
+
name
|
|
73
|
+
modelId
|
|
74
|
+
taskId
|
|
75
|
+
systemPrompt
|
|
76
|
+
promptExamples {
|
|
77
|
+
input
|
|
78
|
+
output
|
|
79
|
+
}
|
|
80
|
+
metaData
|
|
81
|
+
createdAt
|
|
82
|
+
updatedAt
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
`,
|
|
86
|
+
variables: {
|
|
87
|
+
taskId
|
|
88
|
+
}
|
|
89
|
+
}));
|
|
90
|
+
|
|
91
|
+
return aiRemoveTask;
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
async updateTask(taskId: string, task: AITask): Promise<AITask> {
|
|
95
|
+
const { aiUpdateTask } = unwrapApolloResult(await this.#apolloClient.mutate({
|
|
96
|
+
mutation: gql`
|
|
97
|
+
mutation AiUpdateTask($taskId: String!, $task: AITaskInput!) {
|
|
98
|
+
aiUpdateTask(taskId: $taskId, task: $task) {
|
|
99
|
+
name
|
|
100
|
+
modelId
|
|
101
|
+
taskId
|
|
102
|
+
systemPrompt
|
|
103
|
+
promptExamples {
|
|
104
|
+
input
|
|
105
|
+
output
|
|
106
|
+
}
|
|
107
|
+
metaData
|
|
108
|
+
createdAt
|
|
109
|
+
updatedAt
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
`,
|
|
113
|
+
variables: {
|
|
114
|
+
taskId,
|
|
115
|
+
task: {
|
|
116
|
+
name: task.name,
|
|
117
|
+
modelId: task.modelId,
|
|
118
|
+
systemPrompt: task.systemPrompt,
|
|
119
|
+
promptExamples: task.promptExamples
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
}));
|
|
123
|
+
|
|
124
|
+
return aiUpdateTask;
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
async modelLoadingStatus(model: string): Promise<AIModelLoadingStatus> {
|
|
128
|
+
const { aiModelLoadingStatus } = unwrapApolloResult(await this.#apolloClient.query({
|
|
129
|
+
query: gql`
|
|
130
|
+
query AiModelLoadingStatus($model: String!) {
|
|
131
|
+
aiModelLoadingStatus(model: $model) {
|
|
132
|
+
model
|
|
133
|
+
status
|
|
134
|
+
progress
|
|
135
|
+
loaded
|
|
136
|
+
downloaded
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
`,
|
|
140
|
+
variables: {
|
|
141
|
+
model
|
|
142
|
+
}
|
|
143
|
+
}));
|
|
144
|
+
|
|
145
|
+
return aiModelLoadingStatus
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
async prompt(taskId: string, prompt: string): Promise<string> {
|
|
149
|
+
const { aiPrompt } = unwrapApolloResult(await this.#apolloClient.mutate({
|
|
150
|
+
mutation: gql`
|
|
151
|
+
mutation AiPrompt($taskId: String!, $prompt: String!) {
|
|
152
|
+
aiPrompt(taskId: $taskId, prompt: $prompt)
|
|
153
|
+
}
|
|
154
|
+
`,
|
|
155
|
+
variables: {
|
|
156
|
+
taskId,
|
|
157
|
+
prompt
|
|
158
|
+
}
|
|
159
|
+
}));
|
|
160
|
+
|
|
161
|
+
return aiPrompt;
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
async embed(modelId: string, text: string): Promise<Array<number>> {
|
|
165
|
+
const { aiEmbed } = unwrapApolloResult(await this.#apolloClient.mutate({
|
|
166
|
+
mutation: gql`
|
|
167
|
+
mutation aiEmbed($modelId: String!, $text: String!) {
|
|
168
|
+
aiEmbed(modelId: $modelId, text: $text)
|
|
169
|
+
}
|
|
170
|
+
`,
|
|
171
|
+
variables: {
|
|
172
|
+
modelId,
|
|
173
|
+
text
|
|
174
|
+
}
|
|
175
|
+
}));
|
|
176
|
+
|
|
177
|
+
const compressed = base64js.toByteArray(aiEmbed);
|
|
178
|
+
|
|
179
|
+
const decompressed = JSON.parse(pako.inflate(compressed, { to: 'string' }));
|
|
180
|
+
|
|
181
|
+
return decompressed;
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
async openTranscriptionStream(modelId: string, streamCallback: (text: string) => void): Promise<string> {
|
|
185
|
+
const { aiOpenTranscriptionStream } = unwrapApolloResult(await this.#apolloClient.mutate({
|
|
186
|
+
mutation: gql`
|
|
187
|
+
mutation AiOpenTranscriptionStream($modelId: String!) {
|
|
188
|
+
aiOpenTranscriptionStream(modelId: $modelId)
|
|
189
|
+
}
|
|
190
|
+
`,
|
|
191
|
+
variables: {
|
|
192
|
+
modelId
|
|
193
|
+
}
|
|
194
|
+
}));
|
|
195
|
+
|
|
196
|
+
const subscription = this.#apolloClient.subscribe({
|
|
197
|
+
query: gql` subscription {
|
|
198
|
+
aiTranscriptionText(streamId: "${aiOpenTranscriptionStream}")
|
|
199
|
+
}`
|
|
200
|
+
}).subscribe({
|
|
201
|
+
next(data) {
|
|
202
|
+
streamCallback(data.data.aiTranscriptionText);
|
|
203
|
+
|
|
204
|
+
return data.data.aiTranscriptionText;
|
|
205
|
+
},
|
|
206
|
+
error(err) {
|
|
207
|
+
console.error(err);
|
|
208
|
+
}
|
|
209
|
+
});
|
|
210
|
+
|
|
211
|
+
this.#transcriptionSubscriptions.set(aiOpenTranscriptionStream, subscription);
|
|
212
|
+
|
|
213
|
+
return aiOpenTranscriptionStream;
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
async closeTranscriptionStream(streamId: string): Promise<void> {
|
|
217
|
+
const { aiCloseTranscriptionStream } = unwrapApolloResult(await this.#apolloClient.mutate({
|
|
218
|
+
mutation: gql`
|
|
219
|
+
mutation aiCloseTranscriptionStream($streamId: String!) {
|
|
220
|
+
aiCloseTranscriptionStream(streamId: $streamId)
|
|
221
|
+
}
|
|
222
|
+
`,
|
|
223
|
+
variables: {
|
|
224
|
+
streamId
|
|
225
|
+
}
|
|
226
|
+
}));
|
|
227
|
+
|
|
228
|
+
const subscription = this.#transcriptionSubscriptions.get(streamId);
|
|
229
|
+
|
|
230
|
+
if (!subscription.closed) {
|
|
231
|
+
subscription.unsubscribe();
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
return aiCloseTranscriptionStream;
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
async feedTranscriptionStream(streamId: string, audio: Float32Array): Promise<void> {
|
|
238
|
+
const { feedTranscriptionStream } = unwrapApolloResult(await this.#apolloClient.mutate({
|
|
239
|
+
mutation: gql`
|
|
240
|
+
mutation AiFeedTranscriptionStream($streamId: String!, $audio: [Float!]!) {
|
|
241
|
+
aiFeedTranscriptionStream(streamId: $streamId, audio: $audio)
|
|
242
|
+
}
|
|
243
|
+
`,
|
|
244
|
+
variables: {
|
|
245
|
+
streamId,
|
|
246
|
+
audio: audio
|
|
247
|
+
}
|
|
248
|
+
}));
|
|
249
|
+
|
|
250
|
+
return feedTranscriptionStream;
|
|
251
|
+
}
|
|
252
|
+
}
|
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
import { Query, Resolver, Mutation, Arg, InputType, Field, Subscription, Float, PubSub} from "type-graphql";
|
|
2
|
+
import { AIModelLoadingStatus, AITask, AITaskInput } from "./Tasks";
|
|
3
|
+
import pako from "pako";
|
|
4
|
+
import base64js from 'base64-js';
|
|
5
|
+
import { AI_TRANSCRIPTION_TEXT_TOPIC } from "../PubSub";
|
|
6
|
+
|
|
7
|
+
let createdAt = Date.now().toString();
|
|
8
|
+
let updatedAt = Date.now().toString();
|
|
9
|
+
|
|
10
|
+
@Resolver()
|
|
11
|
+
export default class AIResolver {
|
|
12
|
+
@Query(returns => [AITask])
|
|
13
|
+
aiTasks(): AITask[] {
|
|
14
|
+
return [new AITask(
|
|
15
|
+
"task1",
|
|
16
|
+
"modelId",
|
|
17
|
+
"task_id",
|
|
18
|
+
"systemPrompt",
|
|
19
|
+
[],
|
|
20
|
+
undefined,
|
|
21
|
+
createdAt,
|
|
22
|
+
updatedAt
|
|
23
|
+
),
|
|
24
|
+
new AITask(
|
|
25
|
+
"task2",
|
|
26
|
+
"modelId",
|
|
27
|
+
"task_id",
|
|
28
|
+
"systemPrompt",
|
|
29
|
+
[],
|
|
30
|
+
undefined,
|
|
31
|
+
createdAt,
|
|
32
|
+
updatedAt
|
|
33
|
+
)]
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
@Mutation(returns => AITask)
|
|
37
|
+
aiAddTask(
|
|
38
|
+
@Arg("task") task: AITaskInput,
|
|
39
|
+
): AITask {
|
|
40
|
+
return new AITask(
|
|
41
|
+
"task_name",
|
|
42
|
+
task.modelId,
|
|
43
|
+
"task_id",
|
|
44
|
+
task.systemPrompt,
|
|
45
|
+
task.promptExamples,
|
|
46
|
+
undefined,
|
|
47
|
+
createdAt,
|
|
48
|
+
updatedAt
|
|
49
|
+
)
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
@Mutation(() => AITask)
|
|
53
|
+
aiRemoveTask(
|
|
54
|
+
@Arg("taskId") taskId: string
|
|
55
|
+
): AITask {
|
|
56
|
+
return new AITask(
|
|
57
|
+
"task_name",
|
|
58
|
+
"model_id",
|
|
59
|
+
taskId,
|
|
60
|
+
"system prompt",
|
|
61
|
+
[],
|
|
62
|
+
undefined,
|
|
63
|
+
createdAt,
|
|
64
|
+
updatedAt
|
|
65
|
+
)
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
@Mutation(() => AITask)
|
|
69
|
+
aiUpdateTask(
|
|
70
|
+
@Arg("taskId") taskId: string,
|
|
71
|
+
@Arg("task") task:AITaskInput,
|
|
72
|
+
): AITask {
|
|
73
|
+
return new AITask(
|
|
74
|
+
task.name,
|
|
75
|
+
task.modelId,
|
|
76
|
+
taskId,
|
|
77
|
+
task.systemPrompt,
|
|
78
|
+
task.promptExamples,
|
|
79
|
+
undefined,
|
|
80
|
+
createdAt,
|
|
81
|
+
updatedAt
|
|
82
|
+
)
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
@Query(() => AIModelLoadingStatus)
|
|
86
|
+
aiModelLoadingStatus(@Arg("model") model: string): AIModelLoadingStatus {
|
|
87
|
+
return new AIModelLoadingStatus(
|
|
88
|
+
model,
|
|
89
|
+
"loaded",
|
|
90
|
+
100.0,
|
|
91
|
+
true,
|
|
92
|
+
true
|
|
93
|
+
)
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
@Mutation(() => String)
|
|
97
|
+
aiPrompt(
|
|
98
|
+
@Arg("taskId") taskId: string,
|
|
99
|
+
@Arg("prompt") input: string
|
|
100
|
+
): string {
|
|
101
|
+
return "output"
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
@Mutation(() => String)
|
|
105
|
+
aiEmbed(
|
|
106
|
+
@Arg("modelId") modelId: string,
|
|
107
|
+
@Arg("text") text: string
|
|
108
|
+
): string {
|
|
109
|
+
const vec = [0, 10, 20, 30];
|
|
110
|
+
const vecString = JSON.stringify(vec);
|
|
111
|
+
const compressed = pako.deflate(vecString);
|
|
112
|
+
const compressedString = base64js.fromByteArray(compressed);
|
|
113
|
+
|
|
114
|
+
return compressedString;
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
@Mutation(() => String)
|
|
118
|
+
aiOpenTranscriptionStream(
|
|
119
|
+
@Arg("modelId") modelId: string
|
|
120
|
+
): string {
|
|
121
|
+
return "streamId"
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
@Mutation(() => String)
|
|
125
|
+
aiCloseTranscriptionStream(
|
|
126
|
+
@Arg("streamId") streamId: string
|
|
127
|
+
): boolean {
|
|
128
|
+
return true
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
@Mutation(() => String)
|
|
132
|
+
aiFeedTranscriptionStream(
|
|
133
|
+
@Arg("streamId") streamId: string,
|
|
134
|
+
@Arg("audio", () => [Float]) audio: number[],
|
|
135
|
+
@PubSub() pubSub: any
|
|
136
|
+
): boolean {
|
|
137
|
+
pubSub.publish(AI_TRANSCRIPTION_TEXT_TOPIC, { streamId });
|
|
138
|
+
return true
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
@Subscription({ topics: AI_TRANSCRIPTION_TEXT_TOPIC, nullable: false })
|
|
142
|
+
aiTranscriptionText(
|
|
143
|
+
@Arg("streamId") streamId: string
|
|
144
|
+
): string {
|
|
145
|
+
return "transcription"
|
|
146
|
+
}
|
|
147
|
+
}
|