@things-factory/ai-client-base 10.1.53 → 10.1.114
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/config/config.development.js +17 -0
- package/config/config.production.js +9 -0
- package/dist-server/service/agentic-loop/agentic-loop.d.ts +24 -1
- package/dist-server/service/agentic-loop/agentic-loop.js +67 -7
- package/dist-server/service/agentic-loop/agentic-loop.js.map +1 -1
- package/dist-server/service/decision/config.d.ts +23 -0
- package/dist-server/service/decision/config.js +39 -0
- package/dist-server/service/decision/config.js.map +1 -0
- package/dist-server/service/decision/decision.d.ts +37 -0
- package/dist-server/service/decision/decision.js +98 -0
- package/dist-server/service/decision/decision.js.map +1 -0
- package/dist-server/service/decision/index.d.ts +3 -0
- package/dist-server/service/decision/index.js +7 -0
- package/dist-server/service/decision/index.js.map +1 -0
- package/dist-server/service/decision/jev.d.ts +5 -0
- package/dist-server/service/decision/jev.js +46 -0
- package/dist-server/service/decision/jev.js.map +1 -0
- package/dist-server/service/index.d.ts +1 -0
- package/dist-server/service/index.js +1 -0
- package/dist-server/service/index.js.map +1 -1
- package/dist-server/service/providers/openai.d.ts +5 -0
- package/dist-server/service/providers/openai.js +5 -1
- package/dist-server/service/providers/openai.js.map +1 -1
- package/dist-server/tsconfig.tsbuildinfo +1 -1
- package/experiments/entity360/README.md +224 -0
- package/experiments/entity360/apps.cjs +103 -0
- package/experiments/entity360/apps.test.cjs +24 -0
- package/experiments/entity360/baseline-apps-results.json +3357 -0
- package/experiments/entity360/baseline-expanded-results.json +5952 -0
- package/experiments/entity360/baseline-results.json +1507 -0
- package/experiments/entity360/expanded.cjs +65 -0
- package/experiments/entity360/fixtures.json +100 -0
- package/experiments/entity360/index.html +44 -0
- package/experiments/entity360/jev-apps-results.json +3440 -0
- package/experiments/entity360/jev-expanded-results.json +6095 -0
- package/experiments/entity360/jev-results.json +1542 -0
- package/experiments/entity360/run.cjs +100 -0
- package/package.json +2 -2
- package/server/service/agentic-loop/agentic-loop.test.ts +123 -0
- package/server/service/agentic-loop/agentic-loop.ts +66 -8
- package/server/service/decision/config.test.ts +55 -0
- package/server/service/decision/config.ts +49 -0
- package/server/service/decision/decision.test.ts +94 -0
- package/server/service/decision/decision.ts +112 -0
- package/server/service/decision/index.ts +3 -0
- package/server/service/decision/jev.ts +40 -0
- package/server/service/index.ts +1 -0
- package/server/service/providers/openai.ts +5 -2
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
// Synthetic-only pilot. Build the package first; never reads application databases or credentials from files.
|
|
3
|
+
const { readFileSync, writeFileSync } = require('node:fs')
|
|
4
|
+
const { join } = require('node:path')
|
|
5
|
+
const { createServer } = require('node:http')
|
|
6
|
+
const { recommendCandidate } = require('../../dist-server/service/decision/decision.js')
|
|
7
|
+
const { createJevDecisionProvider } = require('../../dist-server/service/decision/jev.js')
|
|
8
|
+
const expanded = process.argv.includes('--expanded')
|
|
9
|
+
const apps = process.argv.includes('--apps')
|
|
10
|
+
if (apps && expanded) throw new Error('Choose one evaluation suite')
|
|
11
|
+
const fixtures = require(apps ? './apps.cjs' : expanded ? './expanded.cjs' : './fixtures.json')
|
|
12
|
+
const live = process.argv.includes('--jev')
|
|
13
|
+
const provider = live ? createJevDecisionProvider(process.env.TYPESAFE_API_KEY || '') : undefined
|
|
14
|
+
const actionProvider = live ? createJevDecisionProvider(process.env.TYPESAFE_API_KEY || '', fetch, 'action') : undefined
|
|
15
|
+
const mode = live ? 'Jev API · synthetic data' : 'Exact baseline · no external model'
|
|
16
|
+
const operationalFailures = new Set(['provider-error', 'timeout', 'invalid-reply', 'invalid-input', 'stale-context', 'access-changed'])
|
|
17
|
+
|
|
18
|
+
async function evaluate(testCase) {
|
|
19
|
+
const candidates = testCase.scenario ? fixtures.scenarios[testCase.scenario] : fixtures.candidates
|
|
20
|
+
const result = await recommendCandidate({
|
|
21
|
+
query: testCase.query, candidates,
|
|
22
|
+
mayRead: async c => candidates.some(f => f.key === c.key && !f.denied),
|
|
23
|
+
isCurrent: () => true, provider: testCase.purpose === 'action' ? actionProvider : provider, timeoutMs: 300
|
|
24
|
+
})
|
|
25
|
+
// An outage must never count as a correct semantic abstention.
|
|
26
|
+
return { ...testCase, result, correct: !operationalFailures.has(result.reason) && (result.candidateKey ?? null) === testCase.expected }
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
function metrics(rows) {
|
|
30
|
+
const selected = rows.filter(r => r.result.status === 'suggested')
|
|
31
|
+
const durations = rows.map(r => r.result.elapsedMs).sort((a, b) => a - b)
|
|
32
|
+
return {
|
|
33
|
+
mode, synthetic: true, cases: rows.length,
|
|
34
|
+
correctIncludingAbstentions: rows.filter(r => r.correct).length,
|
|
35
|
+
suggestions: selected.length, wrongSuggestions: selected.filter(r => !r.correct).length,
|
|
36
|
+
expectedResolvable: rows.filter(r => r.expected).length,
|
|
37
|
+
correctAbstentions: rows.filter(r => !r.expected && r.correct).length,
|
|
38
|
+
missedResolvable: rows.filter(r => r.expected && r.result.status === 'abstained').length,
|
|
39
|
+
errorsOrTimeouts: rows.filter(r => operationalFailures.has(r.result.reason)).length,
|
|
40
|
+
p50Ms: durations[Math.ceil(durations.length * 0.5) - 1],
|
|
41
|
+
p95Ms: durations[Math.ceil(durations.length * 0.95) - 1],
|
|
42
|
+
note: 'Synthetic fixtures on a fixed shortlist; single sequential run, not a production benchmark. Thresholds are uncalibrated.'
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
async function main() {
|
|
47
|
+
if (!process.argv.includes('--serve')) {
|
|
48
|
+
const rows = []
|
|
49
|
+
for (const c of fixtures.cases) rows.push(await evaluate(c))
|
|
50
|
+
const families = [...new Set(rows.map(r => r.baseId || r.id))].map(id => rows.filter(r => (r.baseId || r.id) === id))
|
|
51
|
+
const report = { measuredAt: new Date().toISOString(), metrics: metrics(rows), orderConsistency: {
|
|
52
|
+
logicalCases: families.length,
|
|
53
|
+
sameOutcomeAcrossVariants: families.filter(list => new Set(list.map(r => operationalFailures.has(r.result.reason) ? r.result.reason : r.result.candidateKey ?? null)).size === 1).length,
|
|
54
|
+
allVariantsCorrect: families.filter(list => list.every(r => r.correct)).length
|
|
55
|
+
}, groups: Object.fromEntries(
|
|
56
|
+
[...new Set(rows.map(r => r.group))].map(group => [group, metrics(rows.filter(r => r.group === group))])
|
|
57
|
+
), rows }
|
|
58
|
+
if (process.argv.includes('--report')) {
|
|
59
|
+
writeFileSync(join(__dirname, `${live ? 'jev' : 'baseline'}${apps ? '-apps' : expanded ? '-expanded' : ''}-results.json`), JSON.stringify(report, null, 2) + '\n')
|
|
60
|
+
}
|
|
61
|
+
console.log(JSON.stringify(process.argv.includes('--summary')
|
|
62
|
+
? { measuredAt: report.measuredAt, metrics: report.metrics, orderConsistency: report.orderConsistency,
|
|
63
|
+
failures: rows.filter(r => !r.correct).map(r => ({ id: r.id, query: r.query, expected: r.expected, actual: r.result.candidateKey ?? null, reason: r.result.reason })) }
|
|
64
|
+
: report, null, 2))
|
|
65
|
+
return
|
|
66
|
+
}
|
|
67
|
+
const server = createServer(async (req, res) => {
|
|
68
|
+
const origin = `http://${req.headers.host}`
|
|
69
|
+
res.setHeader('Cache-Control', 'no-store')
|
|
70
|
+
res.setHeader('X-Content-Type-Options', 'nosniff')
|
|
71
|
+
if (req.headers.host !== `127.0.0.1:${server.address().port}` || (req.headers.origin && req.headers.origin !== origin)) {
|
|
72
|
+
res.writeHead(403).end(); return
|
|
73
|
+
}
|
|
74
|
+
if (req.method === 'GET' && req.url === '/') {
|
|
75
|
+
res.setHeader('Content-Type', 'text/html; charset=utf-8')
|
|
76
|
+
res.end(readFileSync(join(__dirname, 'index.html'))); return
|
|
77
|
+
}
|
|
78
|
+
if (req.method === 'GET' && req.url === '/fixtures') {
|
|
79
|
+
res.setHeader('Content-Type', 'application/json')
|
|
80
|
+
res.end(JSON.stringify({ mode, cases: fixtures.cases, candidates: fixtures.candidates.filter(c => !c.denied),
|
|
81
|
+
scenarios: Object.fromEntries(Object.entries(fixtures.scenarios).map(([key, values]) => [key, values.filter(c => !c.denied)])) })); return
|
|
82
|
+
}
|
|
83
|
+
if (req.method !== 'POST' || req.url !== '/evaluate' || req.headers['content-type'] !== 'application/json') {
|
|
84
|
+
res.writeHead(404).end(); return
|
|
85
|
+
}
|
|
86
|
+
try {
|
|
87
|
+
let body = ''
|
|
88
|
+
for await (const chunk of req) {
|
|
89
|
+
body += chunk
|
|
90
|
+
if (body.length > 1024) { res.writeHead(413).end(); return }
|
|
91
|
+
}
|
|
92
|
+
const testCase = fixtures.cases.find(c => c.id === JSON.parse(body).id)
|
|
93
|
+
if (!testCase) { res.writeHead(400).end(); return }
|
|
94
|
+
res.setHeader('Content-Type', 'application/json')
|
|
95
|
+
res.end(JSON.stringify(await evaluate(testCase)))
|
|
96
|
+
} catch { res.writeHead(500).end('Pilot evaluation failed') }
|
|
97
|
+
})
|
|
98
|
+
server.listen(0, '127.0.0.1', () => console.log(JSON.stringify({ url: `http://127.0.0.1:${server.address().port}`, mode })))
|
|
99
|
+
}
|
|
100
|
+
main().catch(() => { console.error('Pilot could not start. Build ai-client-base; --jev also requires TYPESAFE_API_KEY.'); process.exitCode = 1 })
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@things-factory/ai-client-base",
|
|
3
|
-
"version": "10.1.
|
|
3
|
+
"version": "10.1.114",
|
|
4
4
|
"main": "dist-server/index.js",
|
|
5
5
|
"things-factory": true,
|
|
6
6
|
"author": "heartyoh",
|
|
@@ -27,5 +27,5 @@
|
|
|
27
27
|
"@things-factory/env": "^10.1.20",
|
|
28
28
|
"openai": "^4.0.0"
|
|
29
29
|
},
|
|
30
|
-
"gitHead": "
|
|
30
|
+
"gitHead": "0d8f9bc0ff0cb50431f2c76cb166aaf3809d3cf7"
|
|
31
31
|
}
|
|
@@ -917,3 +917,126 @@ describe('runAgenticLoop — 같은 조치는 한 번만 제안된다', () => {
|
|
|
917
917
|
expect(r.proposals).toHaveLength(2)
|
|
918
918
|
})
|
|
919
919
|
})
|
|
920
|
+
|
|
921
|
+
describe('a read tool that hands back a picture (design/ai/shared-dock.md §9-4)', () => {
|
|
922
|
+
const png = Buffer.from([0x89, 0x50, 0x4e, 0x47, 1, 2, 3])
|
|
923
|
+
|
|
924
|
+
test('★ the picture rides as an image part after the results - never in the JSON, the grounding or the usage', async () => {
|
|
925
|
+
const sent: any[][] = []
|
|
926
|
+
let turn = 0
|
|
927
|
+
const input = mkInput(
|
|
928
|
+
{
|
|
929
|
+
chat: async (conversation: any) => {
|
|
930
|
+
sent.push(conversation)
|
|
931
|
+
return turn++ === 0 ? okResult(undefined, [TC('readChatAttachment', { attachmentId: 'a1' })]) : okResult('영수증 금액은 12,400원입니다')
|
|
932
|
+
}
|
|
933
|
+
},
|
|
934
|
+
{
|
|
935
|
+
isReadTool: name => name === 'readChatAttachment',
|
|
936
|
+
executeReadTool: () => ({ attachmentId: 'a1', type: 'image', images: [{ data: png, mediaType: 'image/png' }] })
|
|
937
|
+
}
|
|
938
|
+
)
|
|
939
|
+
const r = await runAgenticLoop(input)
|
|
940
|
+
const last = sent[1][sent[1].length - 1]
|
|
941
|
+
expect(last.role).toBe('user')
|
|
942
|
+
expect(last.content.map((part: any) => part.type)).toEqual(['tool_result', 'image'])
|
|
943
|
+
expect(last.content[1].source).toEqual({ kind: 'base64', data: png, mediaType: 'image/png' })
|
|
944
|
+
expect(JSON.parse(last.content[0].content)).toEqual({ attachmentId: 'a1', type: 'image', imagesAttached: 1 })
|
|
945
|
+
expect(JSON.stringify(r.groundedTexts)).not.toContain('"images":')
|
|
946
|
+
expect(JSON.stringify(r.groundedTexts)).not.toContain('mediaType')
|
|
947
|
+
expect(JSON.stringify(r.toolUsages)).not.toContain('"images":')
|
|
948
|
+
})
|
|
949
|
+
|
|
950
|
+
test('a picture of a type every provider does not take, or with no bytes, is left out and said', async () => {
|
|
951
|
+
const { takeToolImages } = await import('./agentic-loop')
|
|
952
|
+
const { value, images } = takeToolImages({ images: [{ data: png, mediaType: 'image/tiff' }, { data: Buffer.alloc(0), mediaType: 'image/png' }, { data: png.toString('base64'), mediaType: 'IMAGE/JPEG' }] })
|
|
953
|
+
expect(images.map(one => (one.source as any).mediaType)).toEqual(['image/jpeg'])
|
|
954
|
+
expect(value).toEqual({ imagesAttached: 1, imagesLeftOut: 2 })
|
|
955
|
+
expect(takeToolImages({ text: 'a' })).toEqual({ value: { text: 'a' }, images: [] })
|
|
956
|
+
})
|
|
957
|
+
|
|
958
|
+
test('★ OpenAI - the tool messages first, then the pictures as one user message (a tool message holds text only)', async () => {
|
|
959
|
+
const { toSDKMessages } = await import('../providers/openai')
|
|
960
|
+
const messages = toSDKMessages({
|
|
961
|
+
role: 'user',
|
|
962
|
+
content: [
|
|
963
|
+
{ type: 'tool_result', toolUseId: 'c1', content: '{"imagesAttached":1}' },
|
|
964
|
+
{ type: 'image', source: { kind: 'base64', data: png, mediaType: 'image/png' } }
|
|
965
|
+
]
|
|
966
|
+
} as any)
|
|
967
|
+
expect(messages.map((one: any) => one.role)).toEqual(['tool', 'user'])
|
|
968
|
+
expect(messages[1].content[0]).toEqual({ type: 'image_url', image_url: { url: `data:image/png;base64,${png.toString('base64')}` } })
|
|
969
|
+
})
|
|
970
|
+
})
|
|
971
|
+
|
|
972
|
+
describe('what only the card needs (design/ai/read-targets.md §4)', () => {
|
|
973
|
+
test('★ a proposal\'s forScreen reaches the card, never the model', async () => {
|
|
974
|
+
const sent: any[][] = []
|
|
975
|
+
let turn = 0
|
|
976
|
+
const rows = Array.from({ length: 300 }, (_, i) => ({ amount: i }))
|
|
977
|
+
const input = mkInput(
|
|
978
|
+
{
|
|
979
|
+
chat: async (conversation: any) => {
|
|
980
|
+
sent.push(conversation)
|
|
981
|
+
return turn++ === 0 ? okResult(undefined, [TC('queryOfficeCashTransaction')]) : okResult('300건입니다')
|
|
982
|
+
}
|
|
983
|
+
},
|
|
984
|
+
{
|
|
985
|
+
isReadTool: () => true,
|
|
986
|
+
executeReadTool: () => ({ proposed: true, kind: 'workspace', count: 300, forScreen: { workspace: { name: 'w', title: 't', blocks: [{ kind: 'table', rows }] } } })
|
|
987
|
+
}
|
|
988
|
+
)
|
|
989
|
+
const r = await runAgenticLoop(input)
|
|
990
|
+
const toModel = JSON.parse(sent[1][sent[1].length - 1].content[0].content)
|
|
991
|
+
expect(toModel).toEqual({ proposed: true, kind: 'workspace', count: 300 })
|
|
992
|
+
expect(r.proposals[0].workspace.blocks[0].rows).toHaveLength(300)
|
|
993
|
+
expect(JSON.stringify(r.toolUsages)).not.toContain('"forScreen"')
|
|
994
|
+
})
|
|
995
|
+
|
|
996
|
+
test('only a proposal carries it - a plain result keeps the field as it is', async () => {
|
|
997
|
+
const { takeForScreen } = await import('./agentic-loop')
|
|
998
|
+
expect(takeForScreen({ forScreen: { a: 1 } })).toEqual({ value: { forScreen: { a: 1 } }, forScreen: {} })
|
|
999
|
+
})
|
|
1000
|
+
})
|
|
1001
|
+
|
|
1002
|
+
/*
|
|
1003
|
+
* A tool that names its proposal keeps that name (2026-10-02, Figure and AI lanes): figure-service gives a UUID the
|
|
1004
|
+
* modeller creates the new figure under, and the effect key written over it refused the run. Folding is by the effect
|
|
1005
|
+
* key, kept apart as `effectKey`.
|
|
1006
|
+
*/
|
|
1007
|
+
describe('runAgenticLoop — a proposal keeps the name its tool gave it', () => {
|
|
1008
|
+
const twice = (results: any[]) => {
|
|
1009
|
+
let call = 0
|
|
1010
|
+
return mkInput(
|
|
1011
|
+
{
|
|
1012
|
+
chat: async () => (++call <= results.length ? okResult(undefined, [TC('proposeFigureCandidate', {}, `t${call}`)]) : okResult('done'))
|
|
1013
|
+
},
|
|
1014
|
+
{ isReadTool: () => true, executeReadTool: () => results[call - 1] }
|
|
1015
|
+
)
|
|
1016
|
+
}
|
|
1017
|
+
const uuid = '3f1c2a9e-7b4d-4c1e-9a2f-5d6e7f8a9b0c'
|
|
1018
|
+
|
|
1019
|
+
test('the tool\'s key stays the key; the effect key goes beside it', async () => {
|
|
1020
|
+
const r = await runAgenticLoop(twice([{ proposed: true, key: uuid, steps: [{ kind: 'rename', name: 'Bin' }] }]))
|
|
1021
|
+
expect(r.proposals).toHaveLength(1)
|
|
1022
|
+
expect(r.proposals[0].key).toBe(uuid)
|
|
1023
|
+
expect(r.proposals[0].effectKey).toMatch(/^proposeFigureCandidate#/)
|
|
1024
|
+
})
|
|
1025
|
+
|
|
1026
|
+
test('the same proposal twice, worded differently, is still one card', async () => {
|
|
1027
|
+
const r = await runAgenticLoop(
|
|
1028
|
+
twice([
|
|
1029
|
+
{ proposed: true, key: uuid, label: 'one', steps: [{ kind: 'rename', name: 'Bin' }] },
|
|
1030
|
+
{ proposed: true, key: uuid, label: 'two', steps: [{ kind: 'rename', name: 'Bin' }] }
|
|
1031
|
+
])
|
|
1032
|
+
)
|
|
1033
|
+
expect(r.proposals).toHaveLength(1)
|
|
1034
|
+
expect(r.toolUsages.map((u: any) => u.outcome)).toEqual(['proposed', 'folded'])
|
|
1035
|
+
})
|
|
1036
|
+
|
|
1037
|
+
test('a tool that gives no key gets the effect key, as before', async () => {
|
|
1038
|
+
const r = await runAgenticLoop(twice([{ proposed: true, values: { capacity: 10 } }]))
|
|
1039
|
+
expect(r.proposals[0].key).toBe(r.proposals[0].effectKey)
|
|
1040
|
+
expect(r.proposals[0].key).toMatch(/^proposeFigureCandidate#/)
|
|
1041
|
+
})
|
|
1042
|
+
})
|
|
@@ -39,7 +39,7 @@
|
|
|
39
39
|
import { createHash } from 'node:crypto'
|
|
40
40
|
|
|
41
41
|
import debug from 'debug'
|
|
42
|
-
import type { AIMessage, AIToolCall, AIToolChatResult } from '../types.js'
|
|
42
|
+
import type { AIImageMediaType, AIImagePart, AIMessage, AIToolCall, AIToolChatResult } from '../types.js'
|
|
43
43
|
import { retryTransient } from '../utils/index.js'
|
|
44
44
|
|
|
45
45
|
const log = debug('things-factory:ai:agentic-loop')
|
|
@@ -534,13 +534,18 @@ async function processToolCalls<TCtx, TOp>(
|
|
|
534
534
|
iter: number
|
|
535
535
|
): Promise<any[]> {
|
|
536
536
|
const toolResultParts: any[] = []
|
|
537
|
+
const toolResultImages: AIImagePart[] = []
|
|
537
538
|
|
|
538
539
|
for (const tc of toolCalls) {
|
|
539
540
|
if (dispatch.isReadTool(tc.name)) {
|
|
540
541
|
/* await 필수 — registry 에 등록된 read/external builder 는 async 일 수 있다(DB·외부 호출).
|
|
541
542
|
* await 없이 JSON.stringify 하면 Promise 가 '{}' 로 직렬화되어 LLM 이 **빈 결과**를 받는다
|
|
542
543
|
* (문서 가져오기·트윈 구조 조회 등 — 무엇인지는 부르는 쪽이 안다). */
|
|
543
|
-
|
|
544
|
+
/* Pictures a read tool hands back ride as image parts after the results - never inside the JSON. */
|
|
545
|
+
const { value: given, images } = takeToolImages(await dispatch.executeReadTool(tc, context))
|
|
546
|
+
toolResultImages.push(...images)
|
|
547
|
+
/* What only the card needs (a table of a thousand rows) goes to the screen, never to the model. */
|
|
548
|
+
const { value, forScreen } = takeForScreen(given)
|
|
544
549
|
/* 제안 규약 — 결과가 `proposed: true` 면 조치 제안이다(도메인은 모른 채 그대로 실어 올린다).
|
|
545
550
|
* 이 채널이 없으면 대화면은 도구 추적(trace)을 뒤져야 하고, 실행 버튼을 만들 근거가 없다. */
|
|
546
551
|
/* 판정(outcome) — 화면이 단계별 배지로 보여줄 값. 조용한 무동작을 남기지 않는다. */
|
|
@@ -549,13 +554,22 @@ async function processToolCalls<TCtx, TOp>(
|
|
|
549
554
|
/* **같은 조치는 한 번만** — 모델이 같은 것을 두 번 제안하는 일이 실제로 있었다(하나는 인자가
|
|
550
555
|
* 비어 있었다). 카드가 둘이면 사용자가 같은 명령을 두 번 실행할 수 있고, 그건 되돌릴 수 없다.
|
|
551
556
|
* 판정은 **효과**(도구·명령·대상·인자)로 한다 — 설명 문구(label·reason)가 달라도 같은 조치다. */
|
|
552
|
-
const candidate = { tool: tc.name, ...(value as any) }
|
|
553
|
-
const
|
|
554
|
-
if (proposals.some(p => p.
|
|
557
|
+
const candidate = { tool: tc.name, ...(value as any), ...forScreen }
|
|
558
|
+
const effectKey = proposalEffectKey(candidate)
|
|
559
|
+
if (proposals.some(p => p.effectKey === effectKey)) {
|
|
555
560
|
outcome = 'folded' // 접힌 사실을 남긴다 — "왜 카드가 하나뿐인가" 가 추적 가능해야 한다
|
|
556
561
|
} else {
|
|
557
|
-
/*
|
|
558
|
-
|
|
562
|
+
/*
|
|
563
|
+
* 키를 함께 실어 보낸다 — 카드가 같은 규칙으로 다시 세지 않게(원본은 하나).
|
|
564
|
+
*
|
|
565
|
+
* **도구가 이름을 준 제안은 그 이름을 쓴다**(2026-10-02, Figure 레인 · AI 레인 합의). figure-service 의
|
|
566
|
+
* 도구는 제안에 UUID `key` 를 붙이고, 모델러는 그 UUID 로 새 도형을 만든다(ADR-0097 5.6절 5항). 효과 키로
|
|
567
|
+
* 덮어쓰면 「제안 식별자가 올바르지 않습니다」로 실행이 막혔다. 같은 조치인지는 효과 키(`effectKey`)로
|
|
568
|
+
* 따로 판정하므로 접기는 그대로다. 도구 `key` 는 **한 대화 안에서 유일해야 한다** — 겹치면 입력안의 「새
|
|
569
|
+
* 카드가 옛 카드를 대신함」 판정과 결말 줄이 섞인다. 이름을 주지 않는 도구(ai-assistant 의 제안 도구들)는
|
|
570
|
+
* 지금처럼 효과 키를 받는다.
|
|
571
|
+
*/
|
|
572
|
+
proposals.push({ ...candidate, key: typeof (candidate as any).key === 'string' && (candidate as any).key ? (candidate as any).key : effectKey, effectKey })
|
|
559
573
|
outcome = 'proposed'
|
|
560
574
|
}
|
|
561
575
|
} else if (value && typeof value === 'object' && ((value as any).rejected === true || (value as any).error)) {
|
|
@@ -656,5 +670,49 @@ async function processToolCalls<TCtx, TOp>(
|
|
|
656
670
|
}
|
|
657
671
|
}
|
|
658
672
|
|
|
659
|
-
|
|
673
|
+
/* Every provider takes a picture after the results of the same turn (design/ai/shared-dock.md §9-4). */
|
|
674
|
+
return [...toolResultParts, ...toolResultImages]
|
|
675
|
+
}
|
|
676
|
+
|
|
677
|
+
/**
|
|
678
|
+
* A proposal's reserved `forScreen` field, taken out of what the model reads and kept for the card.
|
|
679
|
+
*
|
|
680
|
+
* A read tool that has queried many rows puts them on the card (the AI 작업판 table) but tells the model only the
|
|
681
|
+
* count, the totals and a few rows - the model does not copy a table it never received, so the card's numbers are
|
|
682
|
+
* the query's own (operato-application/design/ai/read-targets.md §4). Only a proposal (`proposed: true`) carries it.
|
|
683
|
+
*/
|
|
684
|
+
export function takeForScreen(result: any): { value: any; forScreen: Record<string, unknown> } {
|
|
685
|
+
if (!result || typeof result !== 'object' || Array.isArray(result) || result.proposed !== true || !result.forScreen || typeof result.forScreen !== 'object') {
|
|
686
|
+
return { value: result, forScreen: {} }
|
|
687
|
+
}
|
|
688
|
+
const { forScreen, ...value } = result
|
|
689
|
+
return { value, forScreen }
|
|
690
|
+
}
|
|
691
|
+
|
|
692
|
+
const TOOL_IMAGE_TYPES = ['image/png', 'image/jpeg', 'image/gif', 'image/webp']
|
|
693
|
+
|
|
694
|
+
/**
|
|
695
|
+
* The pictures a read tool returned in its reserved `images` field, taken out of the result.
|
|
696
|
+
*
|
|
697
|
+
* A tool result reaches the model as JSON text; a picture cannot. So a read tool that must show one - a picture
|
|
698
|
+
* sent in an earlier turn of the conversation - returns `images: [{ data, mediaType }]`, and the loop sends each as an
|
|
699
|
+
* image part of the same turn. The bytes never stay in the JSON, the grounding texts or the usage record. A picture
|
|
700
|
+
* of a type the providers do not all take is left out and said in the result.
|
|
701
|
+
*/
|
|
702
|
+
export function takeToolImages(result: any): { value: any; images: AIImagePart[] } {
|
|
703
|
+
if (!result || typeof result !== 'object' || Array.isArray(result) || !Array.isArray(result.images)) return { value: result, images: [] }
|
|
704
|
+
const { images: given, ...value } = result
|
|
705
|
+
const images: AIImagePart[] = []
|
|
706
|
+
let left = 0
|
|
707
|
+
for (const one of given) {
|
|
708
|
+
const data = typeof one?.data === 'string' ? Buffer.from(one.data, 'base64') : one?.data
|
|
709
|
+
const mediaType = String(one?.mediaType ?? '').toLowerCase()
|
|
710
|
+
if ((Buffer.isBuffer(data) || data instanceof Uint8Array) && data.length && TOOL_IMAGE_TYPES.includes(mediaType)) {
|
|
711
|
+
images.push({ type: 'image', source: { kind: 'base64', data, mediaType: mediaType as AIImageMediaType } })
|
|
712
|
+
} else left++
|
|
713
|
+
}
|
|
714
|
+
return {
|
|
715
|
+
value: { ...value, imagesAttached: images.length, ...(left ? { imagesLeftOut: left } : {}) },
|
|
716
|
+
images
|
|
717
|
+
}
|
|
660
718
|
}
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
const mockGet = jest.fn()
|
|
2
|
+
jest.mock('@things-factory/env', () => ({ config: { get: (...args: any[]) => mockGet(...args) } }))
|
|
3
|
+
import { createConfiguredDecisionClient, getDefaultDecisionClient } from './config'
|
|
4
|
+
|
|
5
|
+
beforeEach(() => mockGet.mockReset())
|
|
6
|
+
|
|
7
|
+
test('absent and disabled config do not require a key or produce a client', () => {
|
|
8
|
+
expect(createConfiguredDecisionClient(null)).toBeUndefined()
|
|
9
|
+
expect(createConfiguredDecisionClient({ enabled: false })).toBeUndefined()
|
|
10
|
+
})
|
|
11
|
+
|
|
12
|
+
test('reads only aiDecision and exposes policy without exposing credentials', () => {
|
|
13
|
+
mockGet.mockReturnValue({ enabled: true, provider: 'typesafe', apiKey: 'test-credential' })
|
|
14
|
+
const client = getDefaultDecisionClient()!
|
|
15
|
+
expect(mockGet).toHaveBeenCalledWith('aiDecision', null)
|
|
16
|
+
expect(client.model).toBe('jev-1.13.0')
|
|
17
|
+
expect(client.timeoutMs).toBe(300)
|
|
18
|
+
expect(JSON.stringify(client)).not.toContain('test-credential')
|
|
19
|
+
})
|
|
20
|
+
|
|
21
|
+
test.each([
|
|
22
|
+
{ enabled: 'true' }, { provider: 'unknown' }, { model: 'jev-latest' },
|
|
23
|
+
{ timeoutMs: 0 }, { timeoutMs: NaN }, { timeoutMs: 10001 }, { apiKey: ' ' }
|
|
24
|
+
])('invalid enabled config fails explicitly without logging supplied values', change => {
|
|
25
|
+
expect(() => createConfiguredDecisionClient({ enabled: true, provider: 'typesafe', apiKey: 'test-credential', ...change } as any)).toThrow(/^aiDecision\./)
|
|
26
|
+
})
|
|
27
|
+
|
|
28
|
+
test('configured deadline governs recommendation even if caller tries to override it', async () => {
|
|
29
|
+
const client = createConfiguredDecisionClient({ enabled: true, provider: 'typesafe', apiKey: 'test-credential', timeoutMs: 5 })!
|
|
30
|
+
const result = await client.recommend({
|
|
31
|
+
query: 'order', candidates: [{ key: '1', label: 'order', description: '' }],
|
|
32
|
+
mayRead: () => new Promise<boolean>(() => {}), isCurrent: () => true, timeoutMs: 10000
|
|
33
|
+
} as any)
|
|
34
|
+
expect(result.reason).toBe('timeout')
|
|
35
|
+
expect(result.provider).toBe('jev-1.13.0')
|
|
36
|
+
})
|
|
37
|
+
|
|
38
|
+
test('host explicitly selects action classification; default stays entity and unknown purpose fails closed', async () => {
|
|
39
|
+
const transport = jest.spyOn(globalThis, 'fetch').mockImplementation(async (_url, options) => {
|
|
40
|
+
const body = JSON.parse(options!.body as string)
|
|
41
|
+
const purpose = Object.keys(body.questions)[0]
|
|
42
|
+
return { ok: true, json: async () => ({ model: 'jev-1.13.0', answers: { [purpose]: {
|
|
43
|
+
type: 'choice', choice: 'read', confidence: 1, probabilities: { read: 1, __none__: 0 }
|
|
44
|
+
} } }) } as Response
|
|
45
|
+
})
|
|
46
|
+
try {
|
|
47
|
+
const client = createConfiguredDecisionClient({ enabled: true, provider: 'typesafe', apiKey: 'test-credential' })!
|
|
48
|
+
const request = { query: 'read', candidates: [{ key: 'read', label: 'read', description: 'Read only' }], mayRead: async () => true, isCurrent: () => true }
|
|
49
|
+
expect((await client.recommend({ ...request, purpose: 'action' })).candidateKey).toBe('read')
|
|
50
|
+
expect((await client.recommend(request)).candidateKey).toBe('read')
|
|
51
|
+
expect(transport.mock.calls.map(([, options]) => Object.keys(JSON.parse(options!.body as string).questions)[0])).toEqual(['action', 'entity'])
|
|
52
|
+
await expect(client.recommend({ ...request, purpose: 'other' as any })).rejects.toThrow('Unsupported decision purpose')
|
|
53
|
+
expect(transport).toHaveBeenCalledTimes(2)
|
|
54
|
+
} finally { transport.mockRestore() }
|
|
55
|
+
})
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
import { config as envConfig } from '@things-factory/env'
|
|
2
|
+
import { recommendCandidate, type DecisionResult } from './decision'
|
|
3
|
+
import { createJevDecisionProvider, JEV_DECISION_MODEL, type JevDecisionPurpose } from './jev'
|
|
4
|
+
|
|
5
|
+
export interface AIDecisionConfig {
|
|
6
|
+
enabled: boolean
|
|
7
|
+
provider?: 'typesafe'
|
|
8
|
+
model?: string
|
|
9
|
+
apiKey?: string
|
|
10
|
+
timeoutMs?: number
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
export type DecisionRequest = Omit<Parameters<typeof recommendCandidate>[0], 'provider' | 'timeoutMs'> & {
|
|
14
|
+
/** Trusted host contract, not a model-selected mode. Defaults to entity selection. */
|
|
15
|
+
purpose?: JevDecisionPurpose
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
export interface DecisionClient {
|
|
19
|
+
readonly provider: 'typesafe'
|
|
20
|
+
readonly model: string
|
|
21
|
+
readonly timeoutMs: number
|
|
22
|
+
recommend(request: DecisionRequest): Promise<DecisionResult>
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
/** Validate once on construction. Errors never include supplied values or secrets. */
|
|
26
|
+
export function createConfiguredDecisionClient(config: AIDecisionConfig | null | undefined): DecisionClient | undefined {
|
|
27
|
+
if (!config || config.enabled === false) return undefined
|
|
28
|
+
if (config.enabled !== true) throw new Error('aiDecision.enabled must be a boolean')
|
|
29
|
+
if (config.provider !== 'typesafe') throw new Error('aiDecision.provider is not supported')
|
|
30
|
+
const model = config.model ?? JEV_DECISION_MODEL
|
|
31
|
+
if (model !== JEV_DECISION_MODEL) throw new Error('aiDecision.model must use the supported pinned version')
|
|
32
|
+
const timeoutMs = config.timeoutMs ?? 300
|
|
33
|
+
if (!Number.isFinite(timeoutMs) || timeoutMs < 1 || timeoutMs > 10000) throw new Error('aiDecision.timeoutMs must be between 1 and 10000')
|
|
34
|
+
if (typeof config.apiKey !== 'string' || !config.apiKey.trim()) throw new Error('aiDecision.apiKey is required when enabled')
|
|
35
|
+
const provider = createJevDecisionProvider(config.apiKey.trim())
|
|
36
|
+
const actionProvider = createJevDecisionProvider(config.apiKey.trim(), fetch, 'action')
|
|
37
|
+
return Object.freeze({
|
|
38
|
+
provider: 'typesafe' as const, model, timeoutMs,
|
|
39
|
+
recommend: async ({ purpose = 'entity', ...request }: DecisionRequest) => {
|
|
40
|
+
if (purpose !== 'entity' && purpose !== 'action') throw new Error('Unsupported decision purpose')
|
|
41
|
+
return recommendCandidate({ ...request, provider: purpose === 'action' ? actionProvider : provider, timeoutMs })
|
|
42
|
+
}
|
|
43
|
+
})
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
/** Server configuration only; constructing this client performs no network request and does not cache credentials. */
|
|
47
|
+
export function getDefaultDecisionClient(): DecisionClient | undefined {
|
|
48
|
+
return createConfiguredDecisionClient(envConfig.get('aiDecision', null) as AIDecisionConfig | null)
|
|
49
|
+
}
|
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
import { recommendCandidate, type DecisionProvider } from './decision'
|
|
2
|
+
import { createJevDecisionProvider } from './jev'
|
|
3
|
+
|
|
4
|
+
const candidates = [
|
|
5
|
+
{ key: 'a', label: '주문 100', description: '서울 출고' },
|
|
6
|
+
{ key: 'b', label: '주문 200', description: '부산 출고' }
|
|
7
|
+
]
|
|
8
|
+
const answer = { choice: 'a', confidence: 0.92, probabilities: { a: 0.94, b: 0.03, __none__: 0.03 } }
|
|
9
|
+
const input = () => ({ query: '주문 100', candidates, mayRead: async () => true, isCurrent: () => true })
|
|
10
|
+
const provider = (reply: unknown): DecisionProvider => ({ name: 'fixture', choose: async () => reply })
|
|
11
|
+
|
|
12
|
+
test('baseline selects exact references, abstains on ambiguity and semantic questions', async () => {
|
|
13
|
+
expect((await recommendCandidate(input())).candidateKey).toBe('a')
|
|
14
|
+
expect((await recommendCandidate({ ...input(), query: '서울로 가는 것' })).reason).toBe('no-match')
|
|
15
|
+
expect((await recommendCandidate({ ...input(), candidates: candidates.map(c => ({ ...c, label: '주문 100' })) })).reason).toBe('no-match')
|
|
16
|
+
})
|
|
17
|
+
|
|
18
|
+
test('permission filtering happens before model exposure, and failure closes access', async () => {
|
|
19
|
+
const choose = jest.fn(async (_q, list) => {
|
|
20
|
+
expect(list.map((c: any) => c.key)).toEqual(['a'])
|
|
21
|
+
return { choice: 'a', confidence: 1, probabilities: { a: 1, __none__: 0 } }
|
|
22
|
+
})
|
|
23
|
+
const result = await recommendCandidate({ ...input(), provider: { name: 'spy', choose }, mayRead: async c => {
|
|
24
|
+
if (c.key === 'b') throw new Error('permission service unavailable')
|
|
25
|
+
return true
|
|
26
|
+
} })
|
|
27
|
+
expect(result.candidateKey).toBe('a')
|
|
28
|
+
expect(result.eligible).toBe(1)
|
|
29
|
+
})
|
|
30
|
+
|
|
31
|
+
test.each([
|
|
32
|
+
{ ...answer, choice: 'invented' },
|
|
33
|
+
{ ...answer, confidence: NaN },
|
|
34
|
+
{ ...answer, probabilities: { a: 1 } },
|
|
35
|
+
{ ...answer, probabilities: { a: 0.4, b: 0.5, __none__: 0.1 } },
|
|
36
|
+
{ ...answer, probabilities: { a: 0.94, b: 0.3, __none__: 0.3 } }
|
|
37
|
+
])('invalid model reply cannot become a target', async reply => {
|
|
38
|
+
expect((await recommendCandidate({ ...input(), provider: provider(reply) })).reason).toBe('invalid-reply')
|
|
39
|
+
})
|
|
40
|
+
|
|
41
|
+
test('uncertain, revoked and stale answers do not recommend a target', async () => {
|
|
42
|
+
expect((await recommendCandidate({ ...input(), provider: provider({ ...answer, confidence: 0.6 }) })).reason).toBe('uncertain')
|
|
43
|
+
let reads = 0
|
|
44
|
+
expect((await recommendCandidate({ ...input(), provider: provider(answer), mayRead: async () => ++reads <= 2 })).reason).toBe('access-changed')
|
|
45
|
+
let current = true
|
|
46
|
+
expect((await recommendCandidate({ ...input(), isCurrent: () => current, provider: { name: 'late', choose: async () => {
|
|
47
|
+
current = false
|
|
48
|
+
return answer
|
|
49
|
+
} } })).reason).toBe('stale-context')
|
|
50
|
+
})
|
|
51
|
+
|
|
52
|
+
test('deadline covers hanging permissions and aborts hanging model requests', async () => {
|
|
53
|
+
expect((await recommendCandidate({ ...input(), timeoutMs: 10, mayRead: () => new Promise(() => {}) })).reason).toBe('timeout')
|
|
54
|
+
let signal: AbortSignal | undefined
|
|
55
|
+
const result = await recommendCandidate({ ...input(), timeoutMs: 10, provider: { name: 'hanging', choose: async (_q, _c, s) => {
|
|
56
|
+
signal = s
|
|
57
|
+
return new Promise(() => {})
|
|
58
|
+
} } })
|
|
59
|
+
expect(result.reason).toBe('timeout')
|
|
60
|
+
expect(signal?.aborted).toBe(true)
|
|
61
|
+
})
|
|
62
|
+
|
|
63
|
+
test('duplicate or oversized shortlist is rejected before provider access', async () => {
|
|
64
|
+
const choose = jest.fn()
|
|
65
|
+
expect((await recommendCandidate({ ...input(), candidates: [candidates[0], candidates[0]], provider: { name: 'spy', choose } })).reason).toBe('invalid-input')
|
|
66
|
+
expect((await recommendCandidate({ ...input(), candidates: Array.from({ length: 13 }, (_, i) => ({ ...candidates[0], key: String(i) })), provider: { name: 'spy', choose } })).reason).toBe('invalid-input')
|
|
67
|
+
expect(choose).not.toHaveBeenCalled()
|
|
68
|
+
})
|
|
69
|
+
|
|
70
|
+
test('Jev uses a pinned typed request and propagates cancellation without returning secrets', async () => {
|
|
71
|
+
const transport = jest.fn(async () => ({ ok: true, json: async () => ({ model: 'jev-1.13.0', answers: { entity: { type: 'choice', ...answer } } }) }))
|
|
72
|
+
const result = await recommendCandidate({ ...input(), provider: createJevDecisionProvider('synthetic-key', transport as any) })
|
|
73
|
+
expect(result.candidateKey).toBe('a')
|
|
74
|
+
const [url, options] = (transport.mock.calls as any)[0]
|
|
75
|
+
expect(url).toBe('https://api.typesafe.ai/v1/systemone')
|
|
76
|
+
const request = JSON.parse(options.body)
|
|
77
|
+
expect(request.model).toBe('jev-1.13.0')
|
|
78
|
+
expect(request.questions.entity.criteria.__none__).toBeTruthy()
|
|
79
|
+
expect(options.signal).toBeInstanceOf(AbortSignal)
|
|
80
|
+
expect(JSON.stringify(result)).not.toContain('synthetic-key')
|
|
81
|
+
})
|
|
82
|
+
|
|
83
|
+
test('action classification uses an explicit contract and never accepts an entity answer', async () => {
|
|
84
|
+
const transport = jest.fn(async () => ({ ok: true, json: async () => ({ model: 'jev-1.13.0', answers: { action: { type: 'choice', ...answer } } }) }))
|
|
85
|
+
const result = await recommendCandidate({ ...input(), provider: createJevDecisionProvider('synthetic-key', transport as any, 'action') })
|
|
86
|
+
expect(result.candidateKey).toBe('a')
|
|
87
|
+
const request = JSON.parse((transport.mock.calls as any)[0][1].body)
|
|
88
|
+
expect(Object.keys(request.questions)).toEqual(['action'])
|
|
89
|
+
expect(request.questions.action.instructions).toContain('never execute')
|
|
90
|
+
expect(request.questions.action.instructions).toContain('multiple workflows')
|
|
91
|
+
const wrongPurpose = async () => ({ ok: true, json: async () => ({ model: 'jev-1.13.0', answers: { entity: { type: 'choice', ...answer } } }) })
|
|
92
|
+
expect((await recommendCandidate({ ...input(), provider: createJevDecisionProvider('synthetic-key', wrongPurpose as any, 'action') })).reason).toBe('provider-error')
|
|
93
|
+
expect(() => createJevDecisionProvider('synthetic-key', transport as any, 'unknown' as any)).toThrow('Unsupported decision purpose')
|
|
94
|
+
})
|