@things-factory/figure-service 10.1.245 → 10.1.246
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +19 -0
- package/dist-server/routers/figure-thumbnail-router.js +16 -1
- package/dist-server/routers/figure-thumbnail-router.js.map +1 -1
- package/dist-server/service/figure/figure-ai-scenarios-run.d.ts +44 -0
- package/dist-server/service/figure/figure-ai-scenarios-run.js +94 -0
- package/dist-server/service/figure/figure-ai-scenarios-run.js.map +1 -0
- package/dist-server/service/figure/figure-ai-scenarios.d.ts +68 -0
- package/dist-server/service/figure/figure-ai-scenarios.js +814 -0
- package/dist-server/service/figure/figure-ai-scenarios.js.map +1 -0
- package/dist-server/service/figure/figure-context.js +3 -0
- package/dist-server/service/figure/figure-context.js.map +1 -1
- package/dist-server/service/figure/figure-llm-provider.d.ts +10 -0
- package/dist-server/service/figure/figure-llm-provider.js +19 -0
- package/dist-server/service/figure/figure-llm-provider.js.map +1 -0
- package/dist-server/service/figure/figure-mutation.js +19 -23
- package/dist-server/service/figure/figure-mutation.js.map +1 -1
- package/dist-server/service/figure/figure-propose-steps.d.ts +5 -1
- package/dist-server/service/figure/figure-propose-steps.js +14 -2
- package/dist-server/service/figure/figure-propose-steps.js.map +1 -1
- package/dist-server/service/figure/figure-query.d.ts +8 -1
- package/dist-server/service/figure/figure-query.js +36 -10
- package/dist-server/service/figure/figure-query.js.map +1 -1
- package/dist-server/tsconfig.tsbuildinfo +1 -1
- package/package.json +8 -8
- package/scripts/figure-ai-scenarios-agent.cjs +98 -0
- package/server/routers/figure-thumbnail-router.ts +18 -2
- package/server/service/figure/figure-ai-scenarios-run.ts +127 -0
- package/server/service/figure/figure-ai-scenarios.live.test.ts +59 -0
- package/server/service/figure/figure-ai-scenarios.test.ts +77 -0
- package/server/service/figure/figure-ai-scenarios.ts +792 -0
- package/server/service/figure/figure-context.ts +3 -0
- package/server/service/figure/figure-llm-provider.ts +20 -0
- package/server/service/figure/figure-llm-smoke.test.ts +5 -68
- package/server/service/figure/figure-mutation.ts +17 -23
- package/server/service/figure/figure-propose-steps.test.ts +55 -0
- package/server/service/figure/figure-propose-steps.ts +14 -3
- package/server/service/figure/figure-public-db.test.ts +108 -0
- package/server/service/figure/figure-query.ts +35 -9
- package/server/service/figure/figure-thumbnail-db.test.ts +16 -0
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@things-factory/figure-service",
|
|
3
|
-
"version": "10.1.
|
|
3
|
+
"version": "10.1.246",
|
|
4
4
|
"publishConfig": {
|
|
5
5
|
"access": "public",
|
|
6
6
|
"@things-factory:registry": "https://registry.npmjs.org"
|
|
@@ -25,18 +25,18 @@
|
|
|
25
25
|
"test:templates": "node --test scripts/templates.test.mts"
|
|
26
26
|
},
|
|
27
27
|
"dependencies": {
|
|
28
|
-
"@hatiolab/figure-model": "^0.12.
|
|
28
|
+
"@hatiolab/figure-model": "^0.12.2",
|
|
29
29
|
"@operato/typeorm-history": "^10.0.0",
|
|
30
|
-
"@things-factory/ai-client-base": "^10.1.
|
|
31
|
-
"@things-factory/attachment-base": "^10.1.
|
|
32
|
-
"@things-factory/auth-base": "^10.1.
|
|
30
|
+
"@things-factory/ai-client-base": "^10.1.246",
|
|
31
|
+
"@things-factory/attachment-base": "^10.1.246",
|
|
32
|
+
"@things-factory/auth-base": "^10.1.246",
|
|
33
33
|
"@things-factory/env": "^10.1.149",
|
|
34
|
-
"@things-factory/setting-base": "^10.1.
|
|
35
|
-
"@things-factory/shell": "^10.1.
|
|
34
|
+
"@things-factory/setting-base": "^10.1.246",
|
|
35
|
+
"@things-factory/shell": "^10.1.246",
|
|
36
36
|
"graphql-scalars": "^1.22.4",
|
|
37
37
|
"graphql-upload": "^15.0.2",
|
|
38
38
|
"jszip": "^3.10.1",
|
|
39
39
|
"koa-router": "^7.4.0"
|
|
40
40
|
},
|
|
41
|
-
"gitHead": "
|
|
41
|
+
"gitHead": "46c0d3c93edf765cc1f13040b1f9d03fd1d9e60b"
|
|
42
42
|
}
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
/**
|
|
3
|
+
* AI 모델링 시나리오를 도메인의 AI 에이전트로 돌린다 (2026-11-03, 사용자 「에이전트 키로」).
|
|
4
|
+
*
|
|
5
|
+
* 앱 폴더에서 실행한다. 그 앱의 설정(데이터 암호 키·DB)을 읽고, 그 DB 의 도메인에 설정된 AI 에이전트를 사람이 쓰는 것과
|
|
6
|
+
* 같은 문(`aiClientFor`)으로 받아, figure-ai-scenarios.ts 의 요청마다 피규어를 만들거나 고치게 한다. 키는 셸에도 저장소에도
|
|
7
|
+
* 두지 않고, 사용량은 그 도메인의 AI 사용 기록에 남는다.
|
|
8
|
+
*
|
|
9
|
+
* cd <앱 폴더, 예 operato-application/packages/operato-office>
|
|
10
|
+
* NODE_ENV=development node <things-factory>/packages/figure-service/scripts/figure-ai-scenarios-agent.cjs --domain hatiolab
|
|
11
|
+
*
|
|
12
|
+
* 선택: --agent <에이전트 id> (없으면 도메인 기본) · --scenarios pipeline-run,hopper · --repeat 3 · --reports <폴더>
|
|
13
|
+
*
|
|
14
|
+
* DB 는 그대로 쓴다: 표를 만들거나 고치지 않는다(synchronize·migration 을 끈다). sqlite 가 아닌 DB 는 --allow-database 를
|
|
15
|
+
* 줄 때만 붙는다 -- 개발 설정이 운영 DB 로 가는 앱(:3300 의 figure)이 있기 때문이다. 운영 모드(NODE_ENV=production)는 받지
|
|
16
|
+
* 않는다. 먼저 figure-service 를 빌드한다: yarn workspace @things-factory/figure-service build:server
|
|
17
|
+
*/
|
|
18
|
+
'use strict'
|
|
19
|
+
|
|
20
|
+
const path = require('path')
|
|
21
|
+
const { createRequire } = require('module')
|
|
22
|
+
|
|
23
|
+
/* things-factory's own packages, whatever the app folder has installed: the entities and the door are the ones built here. */
|
|
24
|
+
const here = createRequire(path.join(__dirname, '..', 'package.json'))
|
|
25
|
+
|
|
26
|
+
function argsOf(argv) {
|
|
27
|
+
const out = { scenarios: [], repeat: 1 }
|
|
28
|
+
for (let i = 0; i < argv.length; i++) {
|
|
29
|
+
const [key, next] = [argv[i], argv[i + 1]]
|
|
30
|
+
if (key === '--domain') (out.domain = next), i++
|
|
31
|
+
else if (key === '--agent') (out.agent = next), i++
|
|
32
|
+
else if (key === '--scenarios') (out.scenarios = next.split(',').map(s => s.trim()).filter(Boolean)), i++
|
|
33
|
+
else if (key === '--repeat') (out.repeat = Math.max(1, Number(next) || 1)), i++
|
|
34
|
+
else if (key === '--reports') (out.reports = path.resolve(next)), i++
|
|
35
|
+
else if (key === '--allow-database') out.allowDatabase = true
|
|
36
|
+
else throw new Error(`unknown argument: ${key}`)
|
|
37
|
+
}
|
|
38
|
+
if (!out.domain) throw new Error('--domain <subdomain> is required: the domain whose AI agent answers')
|
|
39
|
+
return out
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
async function main() {
|
|
43
|
+
const args = argsOf(process.argv.slice(2))
|
|
44
|
+
if (process.env.NODE_ENV === 'production') throw new Error('not in production mode: run it against a development database')
|
|
45
|
+
|
|
46
|
+
const { config } = here('@things-factory/env')
|
|
47
|
+
const { DataSource } = here('typeorm')
|
|
48
|
+
const shell = here('@things-factory/shell')
|
|
49
|
+
const ai = here('@things-factory/ai-client-base')
|
|
50
|
+
const entities = [...shell.entities, ...here('@things-factory/auth-base').entities, ...here('@things-factory/setting-base').entities, ...ai.entities]
|
|
51
|
+
|
|
52
|
+
/* The app's database as its configuration names it; the package default is sqlite db.sqlite in the app folder. */
|
|
53
|
+
const configured = config.get('ormconfig', {}) || {}
|
|
54
|
+
const database = { type: 'sqlite', database: 'db.sqlite', ...configured }
|
|
55
|
+
if (database.type !== 'sqlite' && !args.allowDatabase)
|
|
56
|
+
throw new Error(`the app's database is ${database.type}${database.host ? ` at ${database.host}:${database.port}` : ''}: pass --allow-database to use it`)
|
|
57
|
+
const source = new DataSource({
|
|
58
|
+
...database,
|
|
59
|
+
...(database.type === 'sqlite' ? { database: path.resolve(database.database) } : {}),
|
|
60
|
+
entities,
|
|
61
|
+
migrations: [],
|
|
62
|
+
subscribers: [],
|
|
63
|
+
synchronize: false,
|
|
64
|
+
migrationsRun: false,
|
|
65
|
+
logging: false,
|
|
66
|
+
namingStrategy: new (here('@things-factory/shell/dist-server/initializers/naming-strategy.js').NamingStrategy)()
|
|
67
|
+
})
|
|
68
|
+
await source.initialize()
|
|
69
|
+
shell.addDataSource('default', source)
|
|
70
|
+
try {
|
|
71
|
+
const domain = await source.getRepository(shell.Domain).findOne({ where: { subdomain: args.domain } })
|
|
72
|
+
if (!domain) throw new Error(`no domain ${args.domain} in ${database.database ?? database.host}`)
|
|
73
|
+
const client = await ai.aiClientFor({ domain, agentId: args.agent ?? null })
|
|
74
|
+
|
|
75
|
+
const run = here('./dist-server/service/figure/figure-ai-scenarios-run.js')
|
|
76
|
+
const scenarios = run.scenariosNamed(args.scenarios)
|
|
77
|
+
const runs = []
|
|
78
|
+
for (const scenario of scenarios)
|
|
79
|
+
for (let round = 1; round <= args.repeat; round++) {
|
|
80
|
+
const one = await run.runFigureAIScenario(client, scenario, round)
|
|
81
|
+
runs.push(one)
|
|
82
|
+
console.log(`${one.passed ? '✔' : '✖'} ${scenario.id}${args.repeat > 1 ? ` #${round}` : ''} (${one.seconds.toFixed(1)}s)${one.passed ? '' : ` — ${one.misses.slice(0, 2).join('; ')}`}`)
|
|
83
|
+
}
|
|
84
|
+
const folder = args.reports ?? path.join(__dirname, '..', 'logs', 'figure-ai-scenarios')
|
|
85
|
+
const { lines, report } = run.writeFigureAIScenarioReport(folder, { by: `agent:${args.domain}/${client.agent.name}`, model: `${client.agent.provider}-${client.agent.model}` }, scenarios, runs, args.repeat)
|
|
86
|
+
console.log(lines.join('\n'))
|
|
87
|
+
process.exitCode = report.passed === report.total ? 0 : 1
|
|
88
|
+
} finally {
|
|
89
|
+
/* The last calls' usage rows are written after their answers: closed before them, they would be lost. */
|
|
90
|
+
await ai.aiUsageRecorded()
|
|
91
|
+
await source.destroy()
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
main().catch(e => {
|
|
96
|
+
console.error(e instanceof Error ? e.message : String(e))
|
|
97
|
+
process.exit(2)
|
|
98
|
+
})
|
|
@@ -9,7 +9,8 @@ import { readFileSync } from 'fs'
|
|
|
9
9
|
import { Figure } from '../service/figure/figure.js'
|
|
10
10
|
import { thumbnailAttachmentOf, thumbnailFiles, type ThumbnailFiles } from '../service/figure/figure-thumbnail-store.js'
|
|
11
11
|
import { recipeTemplates } from '../service/figure-recipe/figure-recipe-templates.js'
|
|
12
|
-
import { readableFigure } from '../service/figure/figure-public.js'
|
|
12
|
+
import { foreignPublic, readableFigure } from '../service/figure/figure-public.js'
|
|
13
|
+
import { FigureHistory } from '../service/figure/figure-history.js'
|
|
13
14
|
|
|
14
15
|
/**
|
|
15
16
|
* 썸네일을 **그림으로** 내보내는 길.
|
|
@@ -66,6 +67,16 @@ figureThumbnailRouter.get('/figure-thumbnail/:id', async (context, next) => {
|
|
|
66
67
|
await sendFigureThumbnail(context, next, context.params.id, domain)
|
|
67
68
|
})
|
|
68
69
|
|
|
70
|
+
/** A public figure's picture as its latest release keeps it, on the figure's domain. */
|
|
71
|
+
async function releasedPictureOf(row: Figure): Promise<Pick<Figure, 'thumbnailHash' | 'thumbnailUpdatedAt' | 'domain'> | null> {
|
|
72
|
+
const release = await getRepository(FigureHistory).findOne({
|
|
73
|
+
where: { originalId: row.id },
|
|
74
|
+
order: { version: 'DESC' },
|
|
75
|
+
select: { id: true, thumbnailHash: true, updatedAt: true }
|
|
76
|
+
})
|
|
77
|
+
return release && { thumbnailHash: release.thumbnailHash, thumbnailUpdatedAt: release.updatedAt, domain: row.domain }
|
|
78
|
+
}
|
|
79
|
+
|
|
69
80
|
/**
|
|
70
81
|
* The picture of a figure the caller may read, by its hash (ADR-0104 decision 6). Exported for tests, which give
|
|
71
82
|
* their own `files`.
|
|
@@ -82,7 +93,12 @@ export async function sendFigureThumbnail(
|
|
|
82
93
|
그림만 404 를 내면, 화면은 「그림이 아직 없다」고 거짓말하게 된다.
|
|
83
94
|
*/
|
|
84
95
|
/* Another domain's public figure is read too (G2): its card is what a reader chooses it by. */
|
|
85
|
-
const
|
|
96
|
+
const row = await readableFigure(domain, id, { id: true, thumbnailHash: true, thumbnailUpdatedAt: true, public: true, domain: { id: true } })
|
|
97
|
+
/*
|
|
98
|
+
Another domain is shown the picture of the latest release, as its list is (2026-11-03 audit 1): the row's picture
|
|
99
|
+
may be of a draft not released.
|
|
100
|
+
*/
|
|
101
|
+
const found = row && foreignPublic(domain, row) ? await releasedPictureOf(row) : row
|
|
86
102
|
if (!found?.thumbnailHash) {
|
|
87
103
|
context.status = 404
|
|
88
104
|
return
|
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* AI 모델링 시나리오를 실제 모델로 돌리고 보고서를 남긴다 — figure-ai-scenarios.ts 의 요청마다 피규어를 만들거나 고치게
|
|
3
|
+
* 하고, 결과를 시나리오의 판정으로 잰다.
|
|
4
|
+
*
|
|
5
|
+
* 모델은 부르는 쪽이 준다: 도메인의 AI 에이전트(`aiClientFor`, scripts/figure-ai-scenarios-agent.cjs) -- 사람이 쓰는
|
|
6
|
+
* 것과 같은 모델·설정·키로, 사용량은 그 도메인에 남는다 -- 또는 공급자 키(figure-ai-scenarios.live.test.ts). 보고서는
|
|
7
|
+
* 같은 모델의 지난 보고서와 견주어 나빠진 것과 나아진 것을 말하고, 실패한 답을 남겨 비용 없이 다시 볼 수 있게 한다.
|
|
8
|
+
*/
|
|
9
|
+
import { mkdirSync, readdirSync, readFileSync, writeFileSync } from 'fs'
|
|
10
|
+
import { join } from 'path'
|
|
11
|
+
|
|
12
|
+
import type { AIClient } from '@things-factory/ai-client-base'
|
|
13
|
+
import { createFigureAsset, inspectFigureAsset } from '@hatiolab/figure-model'
|
|
14
|
+
|
|
15
|
+
import { FIGURE_AI_SCENARIOS, type FigureAIScenario } from './figure-ai-scenarios'
|
|
16
|
+
import { ProposeFigureFailure, proposeFigureSteps } from './figure-propose-steps'
|
|
17
|
+
|
|
18
|
+
export interface FigureAIScenarioRun {
|
|
19
|
+
id: string
|
|
20
|
+
title: string
|
|
21
|
+
round: number
|
|
22
|
+
passed: boolean
|
|
23
|
+
/** What the figure missed, or why none was made. */
|
|
24
|
+
misses: string[]
|
|
25
|
+
attempts?: number
|
|
26
|
+
steps?: number
|
|
27
|
+
seconds: number
|
|
28
|
+
/** The steps the model answered -- the last one refused when none was taken -- to look at again without asking it. */
|
|
29
|
+
answer?: unknown[]
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
export interface FigureAIScenarioReport {
|
|
33
|
+
at: string
|
|
34
|
+
/** Who answered: `agent:<domain>/<name>` or the provider, and its model. */
|
|
35
|
+
by: string
|
|
36
|
+
model: string
|
|
37
|
+
rounds: number
|
|
38
|
+
passed: number
|
|
39
|
+
total: number
|
|
40
|
+
comparedWith?: string
|
|
41
|
+
worse: string[]
|
|
42
|
+
better: string[]
|
|
43
|
+
runs: FigureAIScenarioRun[]
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
/** The scenarios a run asks: all, or those named. */
|
|
47
|
+
export const scenariosNamed = (names: readonly string[]) => (names.length ? FIGURE_AI_SCENARIOS.filter(s => names.includes(s.id)) : FIGURE_AI_SCENARIOS)
|
|
48
|
+
|
|
49
|
+
/** One scenario asked once: the figure made, judged by its check and the gate; a failure to make one said as it failed. */
|
|
50
|
+
export async function runFigureAIScenario(client: AIClient, scenario: FigureAIScenario, round = 1): Promise<FigureAIScenarioRun> {
|
|
51
|
+
const started = Date.now()
|
|
52
|
+
const base = scenario.base?.()
|
|
53
|
+
const at = { id: scenario.id, title: scenario.title, round }
|
|
54
|
+
try {
|
|
55
|
+
const made = await proposeFigureSteps({ client, prompt: scenario.prompt, ...(base ? { base } : { name: scenario.name ?? 'Figure' }) })
|
|
56
|
+
const gate = inspectFigureAsset(made.source).violations.map(v => `gate: ${(v as { code?: string }).code ?? JSON.stringify(v)}`)
|
|
57
|
+
const misses = [...gate, ...scenario.check(made.source, base ?? createFigureAsset(scenario.name ?? 'Figure'))]
|
|
58
|
+
return { ...at, passed: !misses.length, misses, attempts: made.attempts, steps: made.steps.length, seconds: (Date.now() - started) / 1000, answer: made.steps }
|
|
59
|
+
} catch (e) {
|
|
60
|
+
const reasons = e instanceof ProposeFigureFailure ? [e.message, ...e.reasons] : [e instanceof Error ? e.message : String(e)]
|
|
61
|
+
const refused = e instanceof ProposeFigureFailure ? (e.lastAnswer as { steps?: unknown[] } | undefined)?.steps : undefined
|
|
62
|
+
return { ...at, passed: false, misses: [`not made: ${reasons.join(' / ')}`], seconds: (Date.now() - started) / 1000, ...(refused ? { answer: refused } : {}) }
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
const fileSafe = (text: string) => text.replace(/[^a-zA-Z0-9.-]/g, '_')
|
|
67
|
+
|
|
68
|
+
/** The last report written by the same answerer and model in `folder`. */
|
|
69
|
+
function lastReport(folder: string, suffix: string): { file: string; runs: FigureAIScenarioRun[] } | undefined {
|
|
70
|
+
try {
|
|
71
|
+
const file = readdirSync(folder)
|
|
72
|
+
.filter(f => f.endsWith(suffix))
|
|
73
|
+
.sort()
|
|
74
|
+
.at(-1)
|
|
75
|
+
return file ? { file, runs: JSON.parse(readFileSync(join(folder, file), 'utf8')).runs } : undefined
|
|
76
|
+
} catch {
|
|
77
|
+
return undefined
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
/**
|
|
82
|
+
* The report of a run, written to `folder` beside the earlier ones and compared with the last of the same answerer and
|
|
83
|
+
* model: a scenario that passed less often is worse, more often better. Returns the report and the lines that say it.
|
|
84
|
+
*/
|
|
85
|
+
export function writeFigureAIScenarioReport(
|
|
86
|
+
folder: string,
|
|
87
|
+
who: { by: string; model: string },
|
|
88
|
+
scenarios: readonly FigureAIScenario[],
|
|
89
|
+
runs: readonly FigureAIScenarioRun[],
|
|
90
|
+
rounds: number
|
|
91
|
+
): { report: FigureAIScenarioReport; file: string; lines: string[] } {
|
|
92
|
+
const suffix = `-${fileSafe(who.by)}-${fileSafe(who.model)}.json`
|
|
93
|
+
const before = lastReport(folder, suffix)
|
|
94
|
+
const rate = (id: string, list: readonly FigureAIScenarioRun[]) => {
|
|
95
|
+
const mine = list.filter(r => r.id === id)
|
|
96
|
+
return mine.length ? mine.filter(r => r.passed).length / mine.length : undefined
|
|
97
|
+
}
|
|
98
|
+
const compared = scenarios.map(s => ({ id: s.id, now: rate(s.id, runs), was: before ? rate(s.id, before.runs) : undefined }))
|
|
99
|
+
const worse = compared.filter(c => c.was !== undefined && c.now !== undefined && c.now < c.was).map(c => c.id)
|
|
100
|
+
const better = compared.filter(c => c.was !== undefined && c.now !== undefined && c.now > c.was).map(c => c.id)
|
|
101
|
+
const report: FigureAIScenarioReport = {
|
|
102
|
+
at: new Date().toISOString(),
|
|
103
|
+
...who,
|
|
104
|
+
rounds,
|
|
105
|
+
passed: runs.filter(r => r.passed).length,
|
|
106
|
+
total: runs.length,
|
|
107
|
+
...(before ? { comparedWith: before.file } : {}),
|
|
108
|
+
worse,
|
|
109
|
+
better,
|
|
110
|
+
runs: [...runs]
|
|
111
|
+
}
|
|
112
|
+
mkdirSync(folder, { recursive: true })
|
|
113
|
+
const file = join(folder, `${report.at.replace(/[:.]/g, '-')}${suffix}`)
|
|
114
|
+
writeFileSync(file, JSON.stringify(report, null, 2))
|
|
115
|
+
const lines = [
|
|
116
|
+
`AI 모델링 시나리오 ${report.passed}/${report.total} 통과 — ${who.by} ${who.model}${rounds > 1 ? ` × ${rounds}` : ''}`,
|
|
117
|
+
...scenarios.map(s => {
|
|
118
|
+
const mine = runs.filter(r => r.id === s.id)
|
|
119
|
+
const ok = mine.filter(r => r.passed).length
|
|
120
|
+
const miss = mine.find(r => !r.passed)?.misses.slice(0, 2).join('; ')
|
|
121
|
+
return ` ${ok === mine.length ? '✔' : '✖'} ${s.id} ${ok}/${mine.length}${miss ? ` — ${miss}` : ''}`
|
|
122
|
+
}),
|
|
123
|
+
...(before ? [` 지난 보고서 ${before.file} 대비 — 나빠짐: ${worse.join(', ') || '없음'} · 나아짐: ${better.join(', ') || '없음'}`] : []),
|
|
124
|
+
` 보고서: ${file}`
|
|
125
|
+
]
|
|
126
|
+
return { report, file, lines }
|
|
127
|
+
}
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* AI 모델링 시나리오를 공급자 키로 (기본 비활성). 도메인의 AI 에이전트로 돌리는 것은 scripts/figure-ai-scenarios-agent.cjs 다.
|
|
3
|
+
*
|
|
4
|
+
* 실행:
|
|
5
|
+
* ANTHROPIC_API_KEY=... yarn test:figure-ai-scenarios
|
|
6
|
+
* 선택: FIGURE_LLM_PROVIDER=anthropic|gemini|openai · FIGURE_LLM_MODEL=<model id>
|
|
7
|
+
* FIGURE_LLM_PROVIDER=reference -- 모델 대신 시나리오의 모범 답을 돌려준다: 키 없이 실행기와 보고서를 점검한다
|
|
8
|
+
* FIGURE_AI_SCENARIOS=pipeline-run,hopper (몇 개만) · FIGURE_AI_REPEAT=3 (같은 요청을 여러 번: 흔들림을 본다)
|
|
9
|
+
* 보고서는 `packages/figure-service/logs/figure-ai-scenarios/` 에 남는다(figure-ai-scenarios-run.ts).
|
|
10
|
+
*/
|
|
11
|
+
import { join } from 'path'
|
|
12
|
+
|
|
13
|
+
import { createAIClient, type AIClient } from '@things-factory/ai-client-base'
|
|
14
|
+
|
|
15
|
+
import type { FigureAIScenario } from './figure-ai-scenarios'
|
|
16
|
+
import { runFigureAIScenario, scenariosNamed, writeFigureAIScenarioReport, type FigureAIScenarioRun } from './figure-ai-scenarios-run'
|
|
17
|
+
import { figureLLMProvider } from './figure-llm-provider'
|
|
18
|
+
|
|
19
|
+
// Provider modules register themselves on import.
|
|
20
|
+
import '@things-factory/ai-client-base'
|
|
21
|
+
|
|
22
|
+
/* `reference`: each scenario answered by its own reference -- the runner and its report checked without a provider. */
|
|
23
|
+
const dry = process.env.FIGURE_LLM_PROVIDER === 'reference'
|
|
24
|
+
const picked = process.env.RUN_FIGURE_AI_SCENARIOS !== '1' ? undefined : dry ? { provider: 'reference', apiKey: '', model: 'reference' } : figureLLMProvider()
|
|
25
|
+
const describeIf = picked ? describe : describe.skip
|
|
26
|
+
|
|
27
|
+
const chosen = scenariosNamed((process.env.FIGURE_AI_SCENARIOS ?? '').split(',').map(s => s.trim()).filter(Boolean))
|
|
28
|
+
const rounds = Math.max(1, Number(process.env.FIGURE_AI_REPEAT ?? 1) || 1)
|
|
29
|
+
const REPORTS = join(__dirname, '..', '..', '..', 'logs', 'figure-ai-scenarios')
|
|
30
|
+
|
|
31
|
+
/** A client that answers the scenario's reference, as a model would answer JSON. */
|
|
32
|
+
const referenceClient = (scenario: FigureAIScenario) => ({ id: 'reference', chat: async () => '', generateJSON: async () => ({ steps: scenario.reference }) }) as unknown as AIClient
|
|
33
|
+
|
|
34
|
+
describeIf('AI 모델링 시나리오 — 공급자 키', () => {
|
|
35
|
+
let client: AIClient
|
|
36
|
+
const runs: FigureAIScenarioRun[] = []
|
|
37
|
+
|
|
38
|
+
beforeAll(() => {
|
|
39
|
+
if (!dry) client = createAIClient(picked as never)
|
|
40
|
+
})
|
|
41
|
+
|
|
42
|
+
afterAll(() => {
|
|
43
|
+
const { lines } = writeFigureAIScenarioReport(REPORTS, { by: picked!.provider, model: picked!.model ?? 'default' }, chosen, runs, rounds)
|
|
44
|
+
// eslint-disable-next-line no-console
|
|
45
|
+
console.log(lines.join('\n'))
|
|
46
|
+
})
|
|
47
|
+
|
|
48
|
+
for (const scenario of chosen)
|
|
49
|
+
for (let round = 1; round <= rounds; round++)
|
|
50
|
+
test(
|
|
51
|
+
`${scenario.id}${rounds > 1 ? ` #${round}` : ''}: ${scenario.title}`,
|
|
52
|
+
async () => {
|
|
53
|
+
const run = await runFigureAIScenario(dry ? referenceClient(scenario) : client, scenario, round)
|
|
54
|
+
runs.push(run)
|
|
55
|
+
expect(run.misses).toEqual([])
|
|
56
|
+
},
|
|
57
|
+
240_000
|
|
58
|
+
)
|
|
59
|
+
})
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* AI 모델링 시나리오의 판정이 맞는지 — 모델 없이 (figure-ai-scenarios.ts).
|
|
3
|
+
*
|
|
4
|
+
* 시나리오마다 판정이 모범 답은 통과시키고, 커널은 받지만 요청을 놓친 답은 떨어뜨리는지 본다. 그래야 실제 모델을 돌린
|
|
5
|
+
* 결과(figure-ai-scenarios.live.test.ts)의 통과·실패를 믿을 수 있다. 커널이나 문법이 바뀌어 모범 답이 더는 받아지지
|
|
6
|
+
* 않으면 여기서 먼저 깨진다.
|
|
7
|
+
*/
|
|
8
|
+
import { createFigureAsset, inspectFigureAsset } from '@hatiolab/figure-model'
|
|
9
|
+
|
|
10
|
+
import { FIGURE_AI_SCENARIOS } from './figure-ai-scenarios'
|
|
11
|
+
import { judgeFigureAnswer } from './figure-propose-steps'
|
|
12
|
+
|
|
13
|
+
const made = (scenario: (typeof FIGURE_AI_SCENARIOS)[number], answer: unknown[]) => {
|
|
14
|
+
const base = scenario.base?.() ?? createFigureAsset(scenario.name ?? 'Figure')
|
|
15
|
+
const judged = judgeFigureAnswer(base, { steps: answer }, !!scenario.base)
|
|
16
|
+
if (!judged.source) throw new Error(`${scenario.id}: the kernel refused the answer: ${judged.reasons.join(' / ')}`)
|
|
17
|
+
return { base, source: judged.source }
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
describe('AI 모델링 시나리오', () => {
|
|
21
|
+
it('each has an id of its own, what it covers, and a request', () => {
|
|
22
|
+
const ids = FIGURE_AI_SCENARIOS.map(s => s.id)
|
|
23
|
+
expect(new Set(ids).size).toBe(ids.length)
|
|
24
|
+
for (const s of FIGURE_AI_SCENARIOS) {
|
|
25
|
+
expect(s.covers.length).toBeGreaterThan(0)
|
|
26
|
+
expect(s.prompt.length).toBeGreaterThan(10)
|
|
27
|
+
}
|
|
28
|
+
})
|
|
29
|
+
|
|
30
|
+
it('they reach the shapes and relations the kernel has: pipes, tapers, joints, rows, clips, data, the figure\'s size', () => {
|
|
31
|
+
const covered = FIGURE_AI_SCENARIOS.flatMap(s => s.covers).join(' ')
|
|
32
|
+
for (const word of ['pipe', 'tapered', 'set-joint', 'make-repeat', 'set-driver', 'set-data-binding', 'follow-size', 'mirror-part', 'set-cavity', 'set-section']) expect(covered).toContain(word)
|
|
33
|
+
expect(FIGURE_AI_SCENARIOS.filter(s => s.base).length).toBeGreaterThanOrEqual(5)
|
|
34
|
+
})
|
|
35
|
+
|
|
36
|
+
it('roll-press: one roller repeated in a row of two, in a colour the frame does not have, is two rollers', () => {
|
|
37
|
+
const scenario = FIGURE_AI_SCENARIOS.find(s => s.id === 'roll-press')!
|
|
38
|
+
const row = (colour: string) => [
|
|
39
|
+
{ kind: 'add-part', part: { id: 'base', kind: 'box', dimensions: { width: 1000, height: 80, depth: 500 } } },
|
|
40
|
+
{ kind: 'add-part', part: { id: 'roller', kind: 'cylinder', dimensions: { radius: 80, length: 400 } } },
|
|
41
|
+
{ kind: 'attach', part: 'base', to: { plane: 'mounting-plane' } },
|
|
42
|
+
{ kind: 'attach', part: 'roller', to: { part: 'base', face: 'top' } },
|
|
43
|
+
{ kind: 'make-repeat', part: 'roller', axis: 'x', pitch: 300, count: 2 },
|
|
44
|
+
{ kind: 'set-material', target: 'roller', material: { token: 'palette.accent' }, palette: { 'palette.accent': colour } }
|
|
45
|
+
]
|
|
46
|
+
const { base, source } = made(scenario, row('#d08a2c'))
|
|
47
|
+
expect(scenario.check(source, base)).toEqual([])
|
|
48
|
+
const plain = made(scenario, row('#d08a2c').slice(0, 4))
|
|
49
|
+
expect(scenario.check(plain.source, plain.base)).toContainEqual(expect.stringMatching(/1 round bodies drawn/))
|
|
50
|
+
})
|
|
51
|
+
|
|
52
|
+
/* Outside review 2026-11-03: a run of 30 passed a request for 300. */
|
|
53
|
+
it('pipe-extend: the bend and the run added are the ones asked, 90° and 300 mm', () => {
|
|
54
|
+
const scenario = FIGURE_AI_SCENARIOS.find(s => s.id === 'pipe-extend')!
|
|
55
|
+
const answer = JSON.parse(JSON.stringify(scenario.reference))
|
|
56
|
+
answer[0].path.at(-1).line = 30
|
|
57
|
+
const short = made(scenario, answer)
|
|
58
|
+
expect(scenario.check(short.source, short.base)).toEqual([expect.stringMatching(/a 300 mm run asked/)])
|
|
59
|
+
answer[0].path.at(-1).line = 300
|
|
60
|
+
answer[0].path.at(-2).bend.angle = 60
|
|
61
|
+
const shallow = made(scenario, answer)
|
|
62
|
+
expect(scenario.check(shallow.source, shallow.base)).toContainEqual(expect.stringMatching(/a 90° bend asked/))
|
|
63
|
+
})
|
|
64
|
+
|
|
65
|
+
describe.each(FIGURE_AI_SCENARIOS.map(s => [s.id, s] as const))('%s', (_, scenario) => {
|
|
66
|
+
it('its reference answer is taken by the kernel and the gate, and passes its check', () => {
|
|
67
|
+
const { base, source } = made(scenario, scenario.reference)
|
|
68
|
+
expect(inspectFigureAsset(source).violations).toEqual([])
|
|
69
|
+
expect(scenario.check(source, base)).toEqual([])
|
|
70
|
+
})
|
|
71
|
+
|
|
72
|
+
it('its wrong answer is taken by the kernel, and its check says what it misses', () => {
|
|
73
|
+
const { base, source } = made(scenario, scenario.wrong)
|
|
74
|
+
expect(scenario.check(source, base).length).toBeGreaterThan(0)
|
|
75
|
+
})
|
|
76
|
+
})
|
|
77
|
+
})
|