oda-engine 0.0.0-stage → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +151 -2
- package/bin/nowhere.mjs +352 -0
- package/lib/account.mjs +57 -0
- package/lib/agentModules.mjs +57 -0
- package/lib/devServer.mjs +351 -0
- package/lib/headless.mjs +231 -0
- package/lib/headlessPreview.mjs +212 -0
- package/lib/keys.mjs +517 -0
- package/lib/make.mjs +855 -0
- package/lib/mcp.mjs +537 -0
- package/lib/open.mjs +22 -0
- package/lib/preview.mjs +87 -0
- package/lib/previewRpc.mjs +134 -0
- package/lib/project.mjs +359 -0
- package/lib/publishFlow.mjs +131 -0
- package/lib/resources.mjs +50 -0
- package/lib/rules.mjs +60 -0
- package/lib/scaffold.mjs +42 -0
- package/lib/setup.mjs +59 -0
- package/lib/tools.mjs +668 -0
- package/package.json +51 -4
- package/template/game.config.json +9 -0
- package/template/scene.json +4 -0
- package/template/src/main.ts +11 -0
- package/vendor/DOCS.md +447 -0
- package/vendor/agent/appRun.mjs +520 -0
- package/vendor/agent/assets/brief.mjs +242 -0
- package/vendor/agent/assets/collider.mjs +531 -0
- package/vendor/agent/assets/environments/index.mjs +12 -0
- package/vendor/agent/assets/environments/worldlabs.mjs +161 -0
- package/vendor/agent/assets/fetch.mjs +168 -0
- package/vendor/agent/assets/footprint.mjs +13 -0
- package/vendor/agent/assets/gltf.mjs +547 -0
- package/vendor/agent/assets/hdr.mjs +269 -0
- package/vendor/agent/assets/imageOps.mjs +171 -0
- package/vendor/agent/assets/images/codex.mjs +307 -0
- package/vendor/agent/assets/images/fal.mjs +75 -0
- package/vendor/agent/assets/images/gemini.mjs +84 -0
- package/vendor/agent/assets/images/index.mjs +25 -0
- package/vendor/agent/assets/images/openai.mjs +75 -0
- package/vendor/agent/assets/images/stability.mjs +71 -0
- package/vendor/agent/assets/images/views.mjs +31 -0
- package/vendor/agent/assets/import.mjs +1352 -0
- package/vendor/agent/assets/inspect.mjs +465 -0
- package/vendor/agent/assets/jobs.mjs +196 -0
- package/vendor/agent/assets/ktx2.mjs +290 -0
- package/vendor/agent/assets/ledger.mjs +466 -0
- package/vendor/agent/assets/net.mjs +254 -0
- package/vendor/agent/assets/prices.json +39 -0
- package/vendor/agent/assets/providers/fal.mjs +115 -0
- package/vendor/agent/assets/providers/index.mjs +22 -0
- package/vendor/agent/assets/providers/meshy.mjs +119 -0
- package/vendor/agent/assets/providers/tripo.mjs +188 -0
- package/vendor/agent/assets/sources/ambientcg.mjs +129 -0
- package/vendor/agent/assets/sources/index.mjs +23 -0
- package/vendor/agent/assets/sources/polyhaven.mjs +237 -0
- package/vendor/agent/assets/sources/polypizza.mjs +121 -0
- package/vendor/agent/assets/splat.mjs +631 -0
- package/vendor/agent/assets/targets.mjs +122 -0
- package/vendor/agent/assets/tile.mjs +261 -0
- package/vendor/agent/assets/unzip.mjs +180 -0
- package/vendor/agent/blender/cleanup.py +360 -0
- package/vendor/agent/blender/convert.py +167 -0
- package/vendor/agent/blender/find.mjs +192 -0
- package/vendor/agent/blender/nw_export.py +104 -0
- package/vendor/agent/blender/nw_rubric.py +426 -0
- package/vendor/agent/blender/run.mjs +193 -0
- package/vendor/agent/blender/runner.py +252 -0
- package/vendor/agent/blender/sheet.py +743 -0
- package/vendor/agent/depth.mjs +134 -0
- package/vendor/agent/director/board.mjs +451 -0
- package/vendor/agent/director/contracts.mjs +663 -0
- package/vendor/agent/director/director.mjs +2365 -0
- package/vendor/agent/director/estimate.mjs +200 -0
- package/vendor/agent/director/gate.mjs +802 -0
- package/vendor/agent/director/limits.json +16 -0
- package/vendor/agent/director/methods.mjs +457 -0
- package/vendor/agent/director/policy.json +33 -0
- package/vendor/agent/director/prompts.mjs +244 -0
- package/vendor/agent/director/reading.mjs +341 -0
- package/vendor/agent/director/router.mjs +184 -0
- package/vendor/agent/director/seams.mjs +243 -0
- package/vendor/agent/director/session.mjs +643 -0
- package/vendor/agent/director/store.mjs +119 -0
- package/vendor/agent/director/taskGate.mjs +110 -0
- package/vendor/agent/director/teamLedger.mjs +118 -0
- package/vendor/agent/director/triage.mjs +63 -0
- package/vendor/agent/downloads.cjs +363 -0
- package/vendor/agent/flow.mjs +78 -0
- package/vendor/agent/guard.mjs +613 -0
- package/vendor/agent/host.mjs +1736 -0
- package/vendor/agent/language.mjs +58 -0
- package/vendor/agent/ledger.mjs +360 -0
- package/vendor/agent/locale.cjs +186 -0
- package/vendor/agent/loop/capacity.mjs +74 -0
- package/vendor/agent/loop/compose.mjs +127 -0
- package/vendor/agent/loop/gate.mjs +205 -0
- package/vendor/agent/loop/judge.mjs +189 -0
- package/vendor/agent/loop/levels.mjs +273 -0
- package/vendor/agent/loop/state.mjs +358 -0
- package/vendor/agent/notes.cjs +278 -0
- package/vendor/agent/pace.mjs +258 -0
- package/vendor/agent/panels.mjs +431 -0
- package/vendor/agent/prompt.mjs +342 -0
- package/vendor/agent/reference/board.mjs +410 -0
- package/vendor/agent/reference/compose.mjs +364 -0
- package/vendor/agent/reference/decode.mjs +56 -0
- package/vendor/agent/reference/facts.mjs +1874 -0
- package/vendor/agent/reference/png.mjs +107 -0
- package/vendor/agent/reference/records.mjs +63 -0
- package/vendor/agent/reference/web.mjs +252 -0
- package/vendor/agent/roles/artist.mjs +69 -0
- package/vendor/agent/roles/builder.mjs +79 -0
- package/vendor/agent/roles/checks.mjs +223 -0
- package/vendor/agent/roles/common.mjs +188 -0
- package/vendor/agent/roles/critic.mjs +171 -0
- package/vendor/agent/roles/index.mjs +39 -0
- package/vendor/agent/roles/modeller.mjs +142 -0
- package/vendor/agent/roles/qa.mjs +58 -0
- package/vendor/agent/roles/rubrics.mjs +79 -0
- package/vendor/agent/roles/sound.mjs +69 -0
- package/vendor/agent/rounds.cjs +333 -0
- package/vendor/agent/shared.mjs +284 -0
- package/vendor/agent/stopGate.mjs +574 -0
- package/vendor/agent/subagents.mjs +129 -0
- package/vendor/agent/tools/assets.mjs +499 -0
- package/vendor/agent/tools/blender.mjs +754 -0
- package/vendor/agent/tools/cards.mjs +479 -0
- package/vendor/agent/tools/choices.mjs +479 -0
- package/vendor/agent/tools/compose.mjs +74 -0
- package/vendor/agent/tools/environment.mjs +488 -0
- package/vendor/agent/tools/eyes.mjs +733 -0
- package/vendor/agent/tools/generate.mjs +338 -0
- package/vendor/agent/tools/guide.mjs +46 -0
- package/vendor/agent/tools/harness.mjs +193 -0
- package/vendor/agent/tools/image.mjs +643 -0
- package/vendor/agent/tools/index.mjs +162 -0
- package/vendor/agent/tools/look.mjs +578 -0
- package/vendor/agent/tools/loop.mjs +858 -0
- package/vendor/agent/tools/others.mjs +123 -0
- package/vendor/agent/tools/pictures.mjs +535 -0
- package/vendor/agent/tools/reference.mjs +366 -0
- package/vendor/agent/transcripts.mjs +118 -0
- package/vendor/agent-loop.mjs +602 -0
- package/vendor/agent-plugin/.claude-plugin/plugin.json +5 -0
- package/vendor/agent-plugin/skills/art-direction/SKILL.md +82 -0
- package/vendor/agent-plugin/skills/asset-legality/SKILL.md +101 -0
- package/vendor/agent-plugin/skills/asset-sources/SKILL.md +111 -0
- package/vendor/agent-plugin/skills/blender-modeling/SKILL.md +97 -0
- package/vendor/agent-plugin/skills/look-dev/SKILL.md +104 -0
- package/vendor/agent-plugin/skills/playtesting/SKILL.md +74 -0
- package/vendor/agent-plugin/skills/reference-loop/SKILL.md +102 -0
- package/vendor/agent-plugin/skills/sound-design/SKILL.md +60 -0
- package/vendor/agent-plugin/skills/style/SKILL.md +35 -0
- package/vendor/agent-plugin/skills/tsl-r185/SKILL.md +107 -0
- package/vendor/auth.cjs +570 -0
- package/vendor/build.cjs +953 -0
- package/vendor/catalog.json +455 -0
- package/vendor/connectors.cjs +688 -0
- package/vendor/eyes.mjs +733 -0
- package/vendor/gameExports.json +1390 -0
- package/vendor/git.cjs +222 -0
- package/vendor/models.json +57 -0
- package/vendor/publish.cjs +1018 -0
- package/vendor/refBoard.cjs +301 -0
- package/vendor/schema/dist/assets.d.ts +335 -0
- package/vendor/schema/dist/assets.js +439 -0
- package/vendor/schema/dist/brief.d.ts +357 -0
- package/vendor/schema/dist/brief.js +302 -0
- package/vendor/schema/dist/common.d.ts +9 -0
- package/vendor/schema/dist/common.js +13 -0
- package/vendor/schema/dist/config.d.ts +406 -0
- package/vendor/schema/dist/config.js +355 -0
- package/vendor/schema/dist/contract.d.ts +599 -0
- package/vendor/schema/dist/contract.js +1146 -0
- package/vendor/schema/dist/index.d.ts +14 -0
- package/vendor/schema/dist/index.js +21 -0
- package/vendor/schema/dist/look.d.ts +463 -0
- package/vendor/schema/dist/look.js +681 -0
- package/vendor/schema/dist/lookStats.d.ts +230 -0
- package/vendor/schema/dist/lookStats.js +768 -0
- package/vendor/schema/dist/pilot.d.ts +32 -0
- package/vendor/schema/dist/pilot.js +44 -0
- package/vendor/schema/dist/publish.d.ts +214 -0
- package/vendor/schema/dist/publish.js +291 -0
- package/vendor/schema/dist/scene.d.ts +75 -0
- package/vendor/schema/dist/scene.js +120 -0
- package/vendor/schema/dist/skyLight.d.ts +141 -0
- package/vendor/schema/dist/skyLight.js +436 -0
- package/vendor/schema/dist/style.d.ts +131 -0
- package/vendor/schema/dist/style.js +155 -0
- package/vendor/schema/package.json +1 -0
- package/vendor/selectionPrompt.mjs +51 -0
- package/vendor/useTools.mjs +195 -0
- package/vendor/userMessage.mjs +26 -0
- package/web-dist/assets/GaussianSplatUtils-CptQkDVX.js +1 -0
- package/web-dist/assets/KTX2Loader-C53LJqss.js +2 -0
- package/web-dist/assets/SPZLoader-BQ3tjaTv.js +1 -0
- package/web-dist/assets/gaussianSplat-_9tcWegB.js +1 -0
- package/web-dist/assets/index-Baka-ZO7.js +693 -0
- package/web-dist/assets/index-CcjyyQco.css +1 -0
- package/web-dist/assets/meshopt_decoder.module-BvwdjiD3.js +1 -0
- package/web-dist/assets/zstddec.module-CfFeQW7d.js +1 -0
- package/web-dist/index.html +34 -0
|
@@ -0,0 +1,643 @@
|
|
|
1
|
+
// image_generate: a picture made for the space, of one of five kinds:
|
|
2
|
+
//
|
|
3
|
+
// object one asset's view before image-to-3D makes the model from it: the
|
|
4
|
+
// object alone (or several views of it), written for this asset
|
|
5
|
+
// from the brief and the reference's crop of its subject. The
|
|
6
|
+
// director chooses when an asset is made this way and with which
|
|
7
|
+
// generator (director/methods.mjs); this tool carries it out.
|
|
8
|
+
// texture a surface that tiles: made seamless here (blended with itself
|
|
9
|
+
// moved by half, tile.mjs) and its 2 x 2 seam score measured
|
|
10
|
+
// before and after
|
|
11
|
+
// panorama a sky: fitted to the 2:1 of an equirectangular picture, its
|
|
12
|
+
// left edge made to meet its right, that seam measured;
|
|
13
|
+
// asset_import as .hdr makes it the config's sky
|
|
14
|
+
// card a cut-out (foliage, a cloud, a distant shape) made on a flat
|
|
15
|
+
// key colour and cut out of it here, with an alpha channel
|
|
16
|
+
// direction one sample of a style direction proposed for the space: the
|
|
17
|
+
// place where a visitor arrives, drawn as that direction draws
|
|
18
|
+
// it, written to refs/style-<name>.png for the style card
|
|
19
|
+
// (nw_ask_pictures purpose style); not one of the space's files
|
|
20
|
+
// target the reference loop's target pictures (REFERENCE_LOOP.md 2.2):
|
|
21
|
+
// the finished space as a visitor would see it from each hero
|
|
22
|
+
// view, one picture for each of 4 to 6 cameras, written from the
|
|
23
|
+
// request and the kickoff's answers in the space's own style
|
|
24
|
+
// (style.json, which must exist first: a photograph only when
|
|
25
|
+
// that is the style) and against the slop list; nw_ask_targets shows
|
|
26
|
+
// them to the creator, and the rounds are judged against them
|
|
27
|
+
// from the same cameras. They are not the space's files: they
|
|
28
|
+
// stay in .nowhere/targets/<n>/ with their record (source.json),
|
|
29
|
+
// out of assets/ and the ledger.
|
|
30
|
+
//
|
|
31
|
+
// A space's scanned libraries come first (asset_search); a made texture is
|
|
32
|
+
// for what they do not have, in the brief's own look.
|
|
33
|
+
//
|
|
34
|
+
// estimate the cost lines for a cost card (nothing is sent)
|
|
35
|
+
// submit an object's pictures into blender/<asset>/ beside the asset's
|
|
36
|
+
// script, each recorded in assets.lock.json as generated: the
|
|
37
|
+
// vendor, the model, the prompt and its hash, and the pictures
|
|
38
|
+
// it started from (a crop of the reference with its subject and
|
|
39
|
+
// box, or an earlier picture with the vendor that made it). A
|
|
40
|
+
// texture, a panorama or a card goes into quarantine with its
|
|
41
|
+
// source.json (the same record), for asset_import to bring into
|
|
42
|
+
// assets/ and the ledger like any download.
|
|
43
|
+
//
|
|
44
|
+
// The generators are in assets/images/: Codex (the creator's ChatGPT plan,
|
|
45
|
+
// after their consent, asked once on a host card), OpenAI, Gemini, fal and
|
|
46
|
+
// Stability (their keys, after a cost card that covers them). Nothing is
|
|
47
|
+
// sent without the consent or the approval, the key read through
|
|
48
|
+
// ctx.secret is never printed, and a picture from a host whose terms forbid
|
|
49
|
+
// feeding it to a model, or the crop of a subject the brief transforms, is
|
|
50
|
+
// never sent. Every check drives the vendors with a fake fetch and Codex
|
|
51
|
+
// with a fake binary.
|
|
52
|
+
|
|
53
|
+
import { z } from 'zod'
|
|
54
|
+
import { createHash, randomBytes } from 'node:crypto'
|
|
55
|
+
import fs from 'node:fs'
|
|
56
|
+
import fsp from 'node:fs/promises'
|
|
57
|
+
import os from 'node:os'
|
|
58
|
+
import path from 'node:path'
|
|
59
|
+
import { IMAGE_PROVIDERS, IMAGE_PROVIDER_IDS, viewPrompts } from '../assets/images/index.mjs'
|
|
60
|
+
import { findCodex, realDeps as codexDeps, runCodex } from '../assets/images/codex.mjs'
|
|
61
|
+
import { addToManifest, slugOf } from '../assets/fetch.mjs'
|
|
62
|
+
import { inputRecordOf, recordAsset, today } from '../assets/ledger.mjs'
|
|
63
|
+
import { fetchOf, imageMime, userAgentFor } from '../assets/net.mjs'
|
|
64
|
+
import { PRICES } from '../assets/providers/index.mjs'
|
|
65
|
+
import { keyOut, makeSeamless, seamScore, tile2x2, toEquirect } from '../assets/tile.mjs'
|
|
66
|
+
import { decodeImage } from '../reference/decode.mjs'
|
|
67
|
+
import { fitEdge } from '../reference/facts.mjs'
|
|
68
|
+
import { encodePng } from '../reference/png.mjs'
|
|
69
|
+
import { pictureInput } from './generate.mjs'
|
|
70
|
+
import { MAX_TARGETS, SLOP } from '../loop/levels.mjs'
|
|
71
|
+
import { readStyle } from '../assets/brief.mjs'
|
|
72
|
+
|
|
73
|
+
export const RPC_METHODS = []
|
|
74
|
+
|
|
75
|
+
const fail = (text) => ({ text, isError: true })
|
|
76
|
+
const usd = (n) => `$${n.toFixed(2)}`
|
|
77
|
+
const slash = (p) => p.split(path.sep).join('/')
|
|
78
|
+
const sha = (text) => createHash('sha256').update(text).digest('hex')
|
|
79
|
+
|
|
80
|
+
function realOrSelf(p) {
|
|
81
|
+
let head = path.resolve(p)
|
|
82
|
+
const rest = []
|
|
83
|
+
for (;;) {
|
|
84
|
+
try {
|
|
85
|
+
return path.join(fs.realpathSync(head), ...rest)
|
|
86
|
+
} catch {
|
|
87
|
+
const up = path.dirname(head)
|
|
88
|
+
if (up === head) return path.resolve(p)
|
|
89
|
+
rest.unshift(path.basename(head))
|
|
90
|
+
head = up
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
/** A project-relative path inside the project, or null. */
|
|
96
|
+
function inProject(root, given) {
|
|
97
|
+
const abs = realOrSelf(path.resolve(root, String(given ?? '')))
|
|
98
|
+
const rel = path.relative(root, abs)
|
|
99
|
+
return !rel || rel.startsWith('..') || path.isAbsolute(rel) ? null : { abs, rel: slash(rel) }
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
/** The files the pictures land in: `out`, then -2, -3, -4 for more views. */
|
|
103
|
+
export function outPaths(out, views) {
|
|
104
|
+
const ext = path.posix.extname(out)
|
|
105
|
+
const stem = out.slice(0, out.length - ext.length)
|
|
106
|
+
return Array.from({ length: views }, (_, k) => (k === 0 ? out : `${stem}-${k + 1}${ext}`))
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
// ---------------------------------------------------------------- the plan
|
|
110
|
+
|
|
111
|
+
/** What a request would run as, or why it cannot run at all. */
|
|
112
|
+
function planFor(args) {
|
|
113
|
+
const provider = IMAGE_PROVIDERS[args.provider]
|
|
114
|
+
if (!provider) return { error: `${args.provider} is not an image generator (${IMAGE_PROVIDER_IDS.join(', ')}).` }
|
|
115
|
+
const inputs = (args.from ?? []).length
|
|
116
|
+
const model = args.model?.trim() || provider.defaultModel({ inputs })
|
|
117
|
+
const refused = provider.refuse?.(model, { inputs })
|
|
118
|
+
if (refused) return { error: refused }
|
|
119
|
+
const views = (args.kind ?? 'object') === 'target' ? Math.max(1, (args.heroes ?? []).length) : (args.views ?? 1)
|
|
120
|
+
const price = provider.price(model, { quality: args.quality, inputs }, PRICES)
|
|
121
|
+
if (!price) return { error: `There is no price for ${provider.label} (${model}) in prices.json, so it cannot be estimated or run.` }
|
|
122
|
+
const heroes = (args.kind ?? 'object') === 'target' ? (args.heroes ?? []) : null
|
|
123
|
+
const lines = Array.from({ length: views }, (_, k) => ({ label: heroes ? `${provider.label} target ${heroes[k]?.id ?? k + 1} (${model}, ${price.unit})` : `${provider.label} picture${views > 1 ? ` ${k + 1} of ${views}` : ''} (${model}, ${price.unit})`, usd: Math.round(price.high * 1000) / 1000 }))
|
|
124
|
+
const totalUsd = Math.round(lines.reduce((n, l) => n + l.usd, 0) * 100) / 100
|
|
125
|
+
const kind = args.kind ?? 'object'
|
|
126
|
+
const id = createHash('sha256').update(JSON.stringify([args.provider, model, args.prompt ?? '', args.from ?? [], views, args.size ?? null, args.quality ?? null, ...(kind === 'object' ? [] : [kind, args.key ?? null]), ...(kind === 'target' ? [args.heroes ?? []] : [])])).digest('hex').slice(0, 12)
|
|
127
|
+
return { provider, model, views, price, lines, totalUsd, estimateId: id }
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
// ---------------------------------------------------------------- the pictures
|
|
131
|
+
|
|
132
|
+
/** A picture as PNG bytes with its size. */
|
|
133
|
+
async function asPng(pic) {
|
|
134
|
+
const buf = Buffer.from(pic.data, 'base64')
|
|
135
|
+
const decoded = await decodeImage(buf, { mime: pic.mime, maxEdge: 4096 })
|
|
136
|
+
if (!decoded.ok) {
|
|
137
|
+
if (imageMime(buf) === 'image/png') return { bytes: buf, width: null, height: null }
|
|
138
|
+
throw new Error(decoded.error)
|
|
139
|
+
}
|
|
140
|
+
return { bytes: imageMime(buf) === 'image/png' ? buf : encodePng(decoded), width: decoded.width, height: decoded.height, decoded }
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
async function writeAtomic(abs, bytes) {
|
|
144
|
+
await fsp.mkdir(path.dirname(abs), { recursive: true })
|
|
145
|
+
const temp = `${abs}.${process.pid}.${randomBytes(3).toString('hex')}.part`
|
|
146
|
+
await fsp.writeFile(temp, bytes)
|
|
147
|
+
await fsp.rename(temp, abs)
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
// ---------------------------------------------------------------- the kinds
|
|
151
|
+
|
|
152
|
+
/** What a kind needs from the generator, in words added to the prompt: how
|
|
153
|
+
* the picture is made, never how it looks (that is the brief's). */
|
|
154
|
+
const KIND_WORDS = {
|
|
155
|
+
texture:
|
|
156
|
+
'Make it a texture that tiles: the surface seen straight on, filling the whole frame edge to edge, flat and even light with no cast shadow or highlight from one side, no perspective, no vignette, no frame, no objects on it, no lettering. Its left edge continues into its right edge, and its top into its bottom.',
|
|
157
|
+
panorama:
|
|
158
|
+
'Make it a 360 degree equirectangular panorama of the sky and the far surroundings, twice as wide as it is tall: the horizon a straight level line across the middle, the top row straight up and the bottom row straight down, the left edge continuing into the right edge. No lettering, no frame.',
|
|
159
|
+
card: (key) =>
|
|
160
|
+
`Show the subject alone and whole, nothing cut off at the edges, on a flat, even, pure ${key} background that touches no part of it, with no shadow on the background and no ${key} in the subject itself. No lettering, no frame.`,
|
|
161
|
+
direction:
|
|
162
|
+
'This is the key picture of one style direction proposed for the space: the place where a visitor arrives, as they would see it on arriving, drawn wholly in the style the words above name, as a finished picture by that hand. A photograph only if that style is one. No lettering, no frame, no watermark, no people in the foreground.',
|
|
163
|
+
}
|
|
164
|
+
const KEY_COLOURS = { green: 'green (#00ff00)', magenta: 'magenta (#ff00ff)', blue: 'blue (#0000ff)' }
|
|
165
|
+
/** The shape each kind is asked in. */
|
|
166
|
+
const KIND_SIZE = { texture: 'square', panorama: 'wide', target: 'wide', direction: 'wide' }
|
|
167
|
+
|
|
168
|
+
/** Whether a style is drawn by the light and materials alone, as a camera
|
|
169
|
+
* sees it: no drawing technique, light as light does. */
|
|
170
|
+
export const isPhotographic = (style) => !!style && style.look === null && style.shading?.kind === 'physical'
|
|
171
|
+
|
|
172
|
+
/** The space's style (style.json) as words for a picture's prompt: how it
|
|
173
|
+
* is drawn, so every target and sample is drawn the same way. */
|
|
174
|
+
export function styleWords(style) {
|
|
175
|
+
if (!style) return ''
|
|
176
|
+
const lines = [
|
|
177
|
+
`Draw it as ${style.medium}.`,
|
|
178
|
+
`Shapes: ${style.shape.language}; ${style.shape.proportions}; ${style.shape.detail} detail.`,
|
|
179
|
+
style.line.kind === 'none' ? 'No drawn lines.' : `Lines: ${style.line.kind}${style.line.weight ? `, ${style.line.weight}` : ''}${style.line.color ? `, in ${style.line.color}` : ''}.`,
|
|
180
|
+
`Shading: ${style.shading.kind}${style.shading.steps ? `, ${style.shading.steps} steps` : ''}${style.shading.shadowColor ? `, shadows toward ${style.shading.shadowColor}` : ''}.`,
|
|
181
|
+
`Surface: ${style.surface}.`,
|
|
182
|
+
`Palette: ${style.palette.colors.join(', ')}; ${style.palette.rule}.`,
|
|
183
|
+
`Rules: ${style.rules.join('; ')}.`,
|
|
184
|
+
...(style.avoid?.length ? [`It must not look like: ${style.avoid.join('; ')}.`] : []),
|
|
185
|
+
]
|
|
186
|
+
return lines.join('\n')
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
/** A target picture's rules (REFERENCE_LOOP.md 2.2): how it is made. How it
|
|
190
|
+
* looks is the space's style (style.json), passed in, and the request's. */
|
|
191
|
+
export function targetWords(hero, avoid = [], style = null) {
|
|
192
|
+
const v3 = (p) => `(${p.map((n) => Number(n).toFixed(1)).join(', ')})`
|
|
193
|
+
const camera = Array.isArray(hero.position) && Array.isArray(hero.lookAt)
|
|
194
|
+
? `The camera stands at ${v3(hero.position)} m and looks toward ${v3(hero.lookAt)} m (y is up)${Number.isFinite(hero.fov) ? `, with a vertical field of view of ${Math.round(hero.fov)} degrees` : ''}.`
|
|
195
|
+
: Number.isFinite(hero.fov)
|
|
196
|
+
? `A vertical field of view of ${Math.round(hero.fov)} degrees.`
|
|
197
|
+
: ''
|
|
198
|
+
return [
|
|
199
|
+
`This is hero view ${hero.id}: ${hero.view}. ${camera}`.trim(),
|
|
200
|
+
isPhotographic(style)
|
|
201
|
+
? 'Make it a photograph of the finished place as a visitor standing in it would see it: realistic, with physically plausible light from one sun and the sky (or the lamps the place has), real materials at their real scale and wear, the camera at a person\'s eye height unless the view says otherwise, and a natural lens with no fisheye, tilt-shift or heavy blur. Not a render, concept art or an illustration.'
|
|
202
|
+
: 'Make it the finished place as a visitor standing in it would see it, drawn in the style below and nothing else: every view of this space by the same hand, the camera at a person\'s eye height unless the view says otherwise, and a natural lens with no fisheye or tilt-shift.',
|
|
203
|
+
styleWords(style),
|
|
204
|
+
`Leave out: ${[...SLOP.filter((s) => !/em dash/.test(s)), ...avoid].join('; ')}. No people in the foreground, no watermark, no frame or border.`,
|
|
205
|
+
].join('\n\n')
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
/** The picture a kind's generation becomes, measured: { pic, file, lines,
|
|
209
|
+
* meta, sheet? }. pic: RGBA. */
|
|
210
|
+
function shapeKind(kind, decoded, name) {
|
|
211
|
+
// What already tiles is kept as it came: the blend would only add its
|
|
212
|
+
// soft ghost of the picture moved by half.
|
|
213
|
+
if (kind === 'texture') {
|
|
214
|
+
const before = seamScore(decoded)
|
|
215
|
+
const pic = before.verdict === 'clean' ? decoded : makeSeamless(decoded)
|
|
216
|
+
const after = before.verdict === 'clean' ? before : seamScore(pic)
|
|
217
|
+
return {
|
|
218
|
+
pic,
|
|
219
|
+
file: `${name}_color.png`,
|
|
220
|
+
lines: [before.verdict === 'clean' ? `As it came it ${before.words}, so it is kept as it came.` : `As it came: ${before.words}. Made seamless here (blended with itself moved by half): ${after.words}.`],
|
|
221
|
+
meta: { kind, seam: { before: before.score, after: after.score, verdict: after.verdict } },
|
|
222
|
+
sheet: tile2x2(pic, 512),
|
|
223
|
+
after,
|
|
224
|
+
}
|
|
225
|
+
}
|
|
226
|
+
if (kind === 'panorama') {
|
|
227
|
+
const fitted = toEquirect(decoded)
|
|
228
|
+
const before = seamScore(fitted, { axes: 'x' })
|
|
229
|
+
const pic = before.verdict === 'clean' ? fitted : makeSeamless(fitted, { axes: 'x', feather: 0.25 })
|
|
230
|
+
const after = before.verdict === 'clean' ? before : seamScore(pic, { axes: 'x' })
|
|
231
|
+
return {
|
|
232
|
+
pic,
|
|
233
|
+
file: `${name}.png`,
|
|
234
|
+
lines: [
|
|
235
|
+
`Fitted to 2:1 (${decoded.width}x${decoded.height} to ${pic.width}x${pic.height}; its full height kept, its width stretched to match): an equirectangular sky has to be, though the generator does not draw a true projection, so look at it wrapped around a sphere before you keep it.`,
|
|
236
|
+
before.verdict === 'clean' ? `Its left edge into its right, as it came: ${before.words}; kept so.` : `Its left edge into its right, as it came: ${before.words}; made to meet: ${after.words}.`,
|
|
237
|
+
],
|
|
238
|
+
meta: { kind, width: pic.width, height: pic.height, seam: { before: before.score, after: after.score, verdict: after.verdict } },
|
|
239
|
+
after,
|
|
240
|
+
}
|
|
241
|
+
}
|
|
242
|
+
const cut = keyOut(decoded)
|
|
243
|
+
return {
|
|
244
|
+
pic: cut.pic,
|
|
245
|
+
file: `${name}.png`,
|
|
246
|
+
lines: [`Cut out of its background (${cut.key}, read from the picture's border): the card covers ${Math.round(cut.coverage * 100)}% of the picture.${cut.problems.length ? ` Look closely: ${cut.problems.join('; ')}.` : ''}`],
|
|
247
|
+
meta: { kind, key: cut.key, coverage: cut.coverage },
|
|
248
|
+
problems: cut.problems,
|
|
249
|
+
}
|
|
250
|
+
}
|
|
251
|
+
|
|
252
|
+
// ---------------------------------------------------------------- image_generate
|
|
253
|
+
|
|
254
|
+
const SHAPE = {
|
|
255
|
+
action: z.enum(['estimate', 'submit']).describe('estimate first (the cost lines, nothing sent); submit once the cost card that covers it was approved (or, for Codex, the creator allowed it).'),
|
|
256
|
+
kind: z
|
|
257
|
+
.enum(['object', 'texture', 'panorama', 'card', 'target', 'direction'])
|
|
258
|
+
.default('object')
|
|
259
|
+
.describe("object: one asset's view for image-to-3D (the default). texture: a surface that tiles, made seamless here. panorama: a 2:1 equirectangular sky. card: a cut-out on a flat key colour, given an alpha channel here. A texture, panorama or card lands in quarantine for asset_import; prefer a scanned library (asset_search) for a real surface, and make one for what it does not have. target: the finished space from each hero view in heroes, one picture each, drawn in style.json\'s style (write it first), shown to the creator before the brief. direction: one sample of a proposed style direction (out: its short name), written to refs/style-<name>.png for the style card."),
|
|
260
|
+
heroes: z
|
|
261
|
+
.array(
|
|
262
|
+
z.object({
|
|
263
|
+
id: z.enum(['H1', 'H2', 'H3', 'H4', 'H5', 'H6']),
|
|
264
|
+
view: z.string().min(1).max(400).describe('In English: where the visitor stands and what this view shows, for this space.'),
|
|
265
|
+
position: z.array(z.number()).length(3).optional().describe('Where the camera stands, metres: the hero view brief.json will take, so the rounds look from the same place.'),
|
|
266
|
+
lookAt: z.array(z.number()).length(3).optional(),
|
|
267
|
+
fov: z.number().min(10).max(120).optional().describe('Vertical field of view, degrees.'),
|
|
268
|
+
}),
|
|
269
|
+
)
|
|
270
|
+
.min(1)
|
|
271
|
+
.max(MAX_TARGETS)
|
|
272
|
+
.optional()
|
|
273
|
+
.describe('target: the hero views to draw, 4 to 6 for a space (fewer to redraw some), each with the camera brief.json\'s heroViews will copy.'),
|
|
274
|
+
avoid: z.array(z.string().min(1).max(200)).max(12).optional().describe("target: what this space also leaves out, beside the loop's own list (brief.json's slop when there is one)."),
|
|
275
|
+
key: z.enum(['green', 'magenta', 'blue']).default('green').describe('card: the flat background it is made on and cut out of. Pick one the subject does not have (magenta for foliage).'),
|
|
276
|
+
provider: z.enum(IMAGE_PROVIDER_IDS).describe("codex: the creator's Codex app on their ChatGPT plan; openai, gemini, fal, stability: their API keys, paid."),
|
|
277
|
+
model: z.string().optional().describe("Leave out for the provider's model in the studio's table."),
|
|
278
|
+
prompt: z.string().min(1).max(4000).describe('What the picture shows, written for this asset from the brief and the reference: the object alone and whole, its form, parts, materials and colours, on a plain background, nothing in front of it, no lettering. target: the place itself from the request and the kickoff\'s answers (what it is, its time of day and weather, its materials and colours), the same words for every hero view.'),
|
|
279
|
+
from: z.array(z.string()).max(4).optional().describe('Project pictures to start from: the reference\'s crop of the subject the task names, or a picture you made before.'),
|
|
280
|
+
views: z.number().int().min(1).max(4).default(1).describe('1: one clean view. 2 to 4: views of the same object (front first, then the right side, the back, the left side), for an image-to-3D service that takes several.'),
|
|
281
|
+
size: z.enum(['square', 'tall', 'wide']).default('square'),
|
|
282
|
+
quality: z.enum(['low', 'medium', 'high']).default('medium').describe('OpenAI\'s quality; the price follows it.'),
|
|
283
|
+
out: z.string().optional().describe('submit, object: where the first picture goes, a .png under blender/<asset>/, e.g. "blender/dome/view.png"; more views end -2, -3, -4. texture, panorama, card: the name it will have in assets/, e.g. "assets/plaster.png" (it waits in quarantine for asset_import).'),
|
|
284
|
+
confirm: z.string().max(40).optional().describe('Only where the host asks for it: the estimate id the estimate answered, once the creator agreed.'),
|
|
285
|
+
}
|
|
286
|
+
|
|
287
|
+
const DESCRIPTION =
|
|
288
|
+
"Makes a picture: kind object (the default) is one asset's view before image-to-3D makes the model from it, one clean view of the object alone, or 2 to 4 views of it (front first), written under blender/<asset>/ and recorded in assets.lock.json (vendor, model, the prompt's hash and the pictures it started from); then asset_generate image-to-3d (one view) or multi-image-to-3d (several) with their paths. kind texture is a surface that tiles (made seamless here, its 2 x 2 seam score before and after, with the 2 x 2 sheet to look at), panorama a 2:1 equirectangular sky (its left edge made to meet its right; asset_import as assets/<name>.hdr makes it the sky), card a cut-out with an alpha channel (made on a flat key colour and cut out here); these land in quarantine with the same record, for asset_import into assets/. kind target draws the finished space from each of 4 to 6 hero views (heroes, with the cameras brief.json will take) as a real photograph would show it, leaving out the slop list, into .nowhere/targets/<n>/ (not the space's files); show them with nw_ask_targets. Codex runs on the creator's ChatGPT plan after their consent; OpenAI, Gemini, fal and Stability are paid, with their keys, after a cost card. action estimate answers the cost lines; submit makes and shows them. Look at them before you go on."
|
|
289
|
+
|
|
290
|
+
async function generate(args, ctx, deps) {
|
|
291
|
+
if (!ctx?.projectRoot) return fail('No project is open.')
|
|
292
|
+
const plan = planFor(args)
|
|
293
|
+
if (plan.error) return fail(plan.error)
|
|
294
|
+
const { provider, model, views, lines, totalUsd, estimateId } = plan
|
|
295
|
+
|
|
296
|
+
if (args.action === 'estimate') {
|
|
297
|
+
const card = { items: lines, note: `${provider.label}, ${PRICES.images?.checked ?? PRICES.checked} prices` }
|
|
298
|
+
if (!provider.paid) {
|
|
299
|
+
return {
|
|
300
|
+
text: [
|
|
301
|
+
`Estimate ${estimateId}: ${views} ${views === 1 ? 'picture' : 'pictures'} with ${provider.label}, on the creator's ${PRICES.images?.codex?.plan ?? 'ChatGPT'} plan: nothing is paid here.`,
|
|
302
|
+
`For the cost card: ${JSON.stringify(card)}`,
|
|
303
|
+
`${provider.label} needs the creator's consent once (a card); submit asks for it when it was not given yet.`,
|
|
304
|
+
].join('\n'),
|
|
305
|
+
}
|
|
306
|
+
}
|
|
307
|
+
return {
|
|
308
|
+
text: [
|
|
309
|
+
`Estimate ${estimateId}: ${lines.map((l) => `${l.label} up to ${usd(l.usd)}`).join('; ')}. Total up to ${usd(totalUsd)} (prices checked ${PRICES.images?.checked ?? PRICES.checked}).`,
|
|
310
|
+
`For the cost card: ${JSON.stringify(card)}`,
|
|
311
|
+
"Ask with nw_ask kind cost, in the screen language, then submit.",
|
|
312
|
+
].join('\n'),
|
|
313
|
+
}
|
|
314
|
+
}
|
|
315
|
+
|
|
316
|
+
// submit: where the pictures go and what they start from, first.
|
|
317
|
+
const root = realOrSelf(ctx.projectRoot)
|
|
318
|
+
const kind = args.kind ?? 'object'
|
|
319
|
+
let targets
|
|
320
|
+
let name = null
|
|
321
|
+
let targetStyle = null
|
|
322
|
+
if (kind === 'target') {
|
|
323
|
+
// Drawn in the space's style, so the style comes first.
|
|
324
|
+
const st = await readStyle(root)
|
|
325
|
+
if (st.status !== 'ok') {
|
|
326
|
+
return fail(
|
|
327
|
+
st.status === 'absent'
|
|
328
|
+
? 'Write style.json first (nw_schema kind "style"): the targets are drawn in the space\'s own style, read from the creator\'s pictures or the style they named, or the direction picked on the style card.'
|
|
329
|
+
: `style.json does not pass its schema, so the targets cannot be drawn in it: ${st.error}`,
|
|
330
|
+
)
|
|
331
|
+
}
|
|
332
|
+
targetStyle = st.style
|
|
333
|
+
const heroes = args.heroes ?? []
|
|
334
|
+
if (heroes.length === 0) return fail('A target needs heroes: the hero views to draw, each with an id (H1 to H6), what it shows and its camera.')
|
|
335
|
+
if (new Set(heroes.map((h) => h.id)).size !== heroes.length) return fail('Each hero view once.')
|
|
336
|
+
const taken = await fsp.readdir(path.join(root, '.nowhere', 'targets')).catch(() => [])
|
|
337
|
+
const n = Math.max(0, ...taken.filter((d) => /^\d{1,3}$/.test(d)).map(Number)) + 1
|
|
338
|
+
targets = heroes.map((h) => `.nowhere/targets/${n}/${h.id}.png`)
|
|
339
|
+
} else if (kind === 'direction') {
|
|
340
|
+
const m = /^(?:refs\/)?(?:style-)?([a-z0-9][a-z0-9-]{0,39})(?:\.png)?$/.exec(String(args.out ?? '').trim())
|
|
341
|
+
if (!m) return fail('A direction needs out: its short name in a-z, 0-9 and -, e.g. "ink-storybook". It is written to refs/style-<name>.png.')
|
|
342
|
+
targets = [`refs/style-${m[1]}.png`]
|
|
343
|
+
} else if (kind === 'object') {
|
|
344
|
+
if (!args.out) return fail('submit needs out: where the first picture goes, a .png under blender/<asset>/, e.g. "blender/dome/view.png".')
|
|
345
|
+
const out = inProject(root, args.out)
|
|
346
|
+
if (!out || !/^blender\/[^/]+\/.+\.png$/i.test(out.rel)) return fail(`out "${args.out}" is not a .png in a folder of its own under blender/, e.g. "blender/dome/view.png".`)
|
|
347
|
+
targets = outPaths(out.rel, views)
|
|
348
|
+
} else {
|
|
349
|
+
if (views > 1) return fail(`views are an object's; a ${kind} is one picture a call (make another for a variant).`)
|
|
350
|
+
const m = /^(?:assets\/)?([A-Za-z0-9][A-Za-z0-9_-]{0,63})(?:\.png)?$/.exec(String(args.out ?? '').trim())
|
|
351
|
+
if (args.out && !m) return fail(`out "${args.out}" is not the name it will have in assets/, e.g. "assets/plaster.png".`)
|
|
352
|
+
name = m ? m[1] : slugOf(args.prompt).slice(0, 40) || kind
|
|
353
|
+
targets = [name]
|
|
354
|
+
}
|
|
355
|
+
// What it starts from, through asset_generate's own checks (pictureInput).
|
|
356
|
+
const inputs = []
|
|
357
|
+
for (const given of args.from ?? []) {
|
|
358
|
+
const picture = await pictureInput(ctx, given)
|
|
359
|
+
if (picture.error) return fail(picture.error.replace(/^image "/, '"'))
|
|
360
|
+
inputs.push(picture)
|
|
361
|
+
}
|
|
362
|
+
|
|
363
|
+
// The consent or the key and the approval. Nothing is sent before them.
|
|
364
|
+
let key = null
|
|
365
|
+
let codexBin = null
|
|
366
|
+
if (provider.id === 'codex') {
|
|
367
|
+
const answer = typeof ctx?.codex === 'function' ? await Promise.resolve(ctx.codex()).catch(() => 'unavailable') : 'unavailable'
|
|
368
|
+
if (answer !== 'allow') {
|
|
369
|
+
return fail(
|
|
370
|
+
answer === 'deny'
|
|
371
|
+
? "The creator has not allowed Codex to make pictures on their ChatGPT plan, so nothing was made. Use a paid generator after a cost card, or make the asset another way."
|
|
372
|
+
: 'There is no answer from the creator about Codex here, so nothing was made.',
|
|
373
|
+
)
|
|
374
|
+
}
|
|
375
|
+
const found = await deps.findCodex()
|
|
376
|
+
if (found.error) return fail(found.error)
|
|
377
|
+
if (found.login !== 'chatgpt') return fail(found.login ? 'Codex is logged in with an API key, which bills that account rather than the creator\'s ChatGPT plan, so nothing was made. Sign Codex in with ChatGPT (codex login), or use a paid generator after a cost card.' : 'Codex is not logged in, so nothing was made: open the ChatGPT app and sign in (or run codex login), then try again.')
|
|
378
|
+
codexBin = found.path
|
|
379
|
+
} else {
|
|
380
|
+
try {
|
|
381
|
+
const v = ctx?.secret?.(provider.envVar)
|
|
382
|
+
key = typeof v === 'string' && v.trim() ? v.trim() : null
|
|
383
|
+
} catch {
|
|
384
|
+
key = null
|
|
385
|
+
}
|
|
386
|
+
if (!key) return fail(`${provider.label} has no key here. Ask for it with nw_ask kind key, connector ${provider.connector}.`)
|
|
387
|
+
if (args.confirm && typeof ctx?.ledger?.confirmEstimate === 'function') {
|
|
388
|
+
try {
|
|
389
|
+
ctx.ledger.confirmEstimate({ id: args.confirm, usd: totalUsd, matches: args.confirm === estimateId })
|
|
390
|
+
} catch {
|
|
391
|
+
// the host's stand-in decides; the approval below is the gate
|
|
392
|
+
}
|
|
393
|
+
}
|
|
394
|
+
const approved = ctx?.ledger?.approvedUsd?.()
|
|
395
|
+
const spent = Number(ctx?.ledger?.spentUsd?.() ?? 0) || 0
|
|
396
|
+
if (approved === null || approved === undefined || !Number.isFinite(approved)) {
|
|
397
|
+
return fail(`No cost card has been approved for this, so nothing was sent. Run action estimate, ask the creator with nw_ask kind cost (${usd(totalUsd)}), and submit after they approve.`)
|
|
398
|
+
}
|
|
399
|
+
if (approved - spent < totalUsd - 1e-9) {
|
|
400
|
+
return fail(`The approval left is ${usd(Math.max(0, approved - spent))} (approved ${usd(approved)}, spent ${usd(spent)}), less than this estimate of ${usd(totalUsd)}, so nothing was sent. Ask again with nw_ask kind cost for the new total.`)
|
|
401
|
+
}
|
|
402
|
+
}
|
|
403
|
+
|
|
404
|
+
const prompts =
|
|
405
|
+
kind === 'target'
|
|
406
|
+
? (args.heroes ?? []).map((h) => `${args.prompt}\n\n${targetWords(h, args.avoid ?? [], targetStyle)}`)
|
|
407
|
+
: kind === 'object'
|
|
408
|
+
? viewPrompts(args.prompt, views)
|
|
409
|
+
: [`${args.prompt}\n\n${kind === 'card' ? KIND_WORDS.card(KEY_COLOURS[args.key ?? 'green']) : KIND_WORDS[kind]}`]
|
|
410
|
+
const jobId = `${provider.id}-img-${today().replace(/-/g, '')}-${randomBytes(3).toString('hex')}`
|
|
411
|
+
const started = deps.now()
|
|
412
|
+
let pictures
|
|
413
|
+
// A paid generator that failed after some views gives those back: they
|
|
414
|
+
// were billed, so they are written, recorded and counted in the spend.
|
|
415
|
+
let failed = null
|
|
416
|
+
try {
|
|
417
|
+
if (codexBin) {
|
|
418
|
+
const work = await fsp.mkdtemp(path.join(ctx.runDir && fs.existsSync(ctx.runDir) ? ctx.runDir : os.tmpdir(), 'codex-'))
|
|
419
|
+
try {
|
|
420
|
+
pictures = await runCodex({ bin: codexBin, prompts, inputs: inputs.map((i) => i.abs), workDir: work, deps: deps.codexDeps })
|
|
421
|
+
} finally {
|
|
422
|
+
await fsp.rm(work, { recursive: true, force: true })
|
|
423
|
+
}
|
|
424
|
+
} else {
|
|
425
|
+
pictures = await provider.generate({ model, prompts, inputs: inputs.map(({ mime, data }) => ({ mime, data })), size: KIND_SIZE[kind] ?? args.size, quality: args.quality, key, fetch: fetchOf(ctx), ua: userAgentFor(ctx) })
|
|
426
|
+
}
|
|
427
|
+
} catch (err) {
|
|
428
|
+
const why = String(err?.message ?? err).slice(0, 400)
|
|
429
|
+
if (!err?.made?.length) return fail(`${provider.label} could not make it, so nothing was written${provider.paid ? ' or spent' : ''}: ${why}`)
|
|
430
|
+
pictures = err.made
|
|
431
|
+
failed = why
|
|
432
|
+
}
|
|
433
|
+
if (!pictures?.length) return fail(`${provider.label} answered with no picture, so nothing was written.`)
|
|
434
|
+
// Recorded once, at the estimate's upper price as asset_generate does, for
|
|
435
|
+
// each picture that came back.
|
|
436
|
+
const back = Math.min(pictures.length, targets.length)
|
|
437
|
+
const spentUsd = Math.round(lines.slice(0, back).reduce((n, l) => n + l.usd, 0) * 100) / 100
|
|
438
|
+
if (provider.paid) {
|
|
439
|
+
try {
|
|
440
|
+
ctx?.ledger?.addGeneration?.(spentUsd, `${provider.id} image`)
|
|
441
|
+
} catch (err) {
|
|
442
|
+
ctx?.log?.(`[image] ledger.addGeneration failed: ${err?.message ?? err}`)
|
|
443
|
+
}
|
|
444
|
+
}
|
|
445
|
+
|
|
446
|
+
const made = []
|
|
447
|
+
const images = []
|
|
448
|
+
const from = inputs.map((i) => inputRecordOf(i.rel, i.sha256, i.entry))
|
|
449
|
+
if (kind === 'target') return await landTargets({ heroes: args.heroes ?? [], targets, pictures: pictures.slice(0, back), prompts, provider, model, jobId, root, lines, spentUsd, failed, seconds: (deps.now() - started) / 1000 })
|
|
450
|
+
if (kind === 'direction') return await landDirection({ rel: targets[0], picture: pictures[0], provider, model, spentUsd, root, seconds: (deps.now() - started) / 1000 })
|
|
451
|
+
if (kind !== 'object') return await landKind({ kind, name, provider, model, prompt: prompts[0], picture: pictures[0], jobId, from, root, ctx, usd: provider.paid ? lines[0].usd : 0, spentUsd, seconds: (deps.now() - started) / 1000 })
|
|
452
|
+
for (let k = 0; k < back; k++) {
|
|
453
|
+
let png
|
|
454
|
+
try {
|
|
455
|
+
png = await asPng(pictures[k])
|
|
456
|
+
} catch (err) {
|
|
457
|
+
return fail(`${provider.label}'s picture ${k + 1} could not be read (${err?.message ?? err}); ${made.length ? `${made.join(', ')} ${made.length === 1 ? 'was' : 'were'} written` : 'nothing was written'}.`)
|
|
458
|
+
}
|
|
459
|
+
const rel = targets[k]
|
|
460
|
+
await writeAtomic(path.join(root, ...rel.split('/')), png.bytes)
|
|
461
|
+
const rec = await recordAsset(root, rel, {
|
|
462
|
+
kind: 'image',
|
|
463
|
+
origin: 'generated',
|
|
464
|
+
license: provider.licenseOf(model),
|
|
465
|
+
source: { provider: provider.id, id: `${jobId}-${k + 1}`, title: args.prompt.slice(0, 200), author: `${provider.label} (generated)`, fetchedAt: today() },
|
|
466
|
+
generation: { vendor: provider.id, model, prompt: prompts[k].slice(0, 2000), job: jobId, usd: provider.paid ? lines[k].usd : 0 },
|
|
467
|
+
...(from.length ? { derivedFrom: from.map((f) => f.path).filter((p) => /^(assets|blender)\//.test(p)) } : {}),
|
|
468
|
+
steps: [{ op: 'image_generate', at: new Date().toISOString(), detail: `view ${k + 1} of ${views}, ${provider.label} ${model}, in ${((deps.now() - started) / 1000).toFixed(1)} s` }],
|
|
469
|
+
// The whole prompt's hash and what it started from, beside what it is.
|
|
470
|
+
meta: { width: png.width, height: png.height, view: k + 1, views, provenance: { promptSha256: sha(prompts[k]), inputs: from } },
|
|
471
|
+
})
|
|
472
|
+
made.push(`${rel}${png.width ? ` (${png.width}x${png.height})` : ''}${rec.ok ? '' : `, not recorded: ${rec.error}`}`)
|
|
473
|
+
if (png.decoded) images.push({ mime: 'image/png', data: encodePng(fitEdge(png.decoded, 768)).toString('base64') })
|
|
474
|
+
}
|
|
475
|
+
const paths = targets.slice(0, made.length)
|
|
476
|
+
const next =
|
|
477
|
+
paths.length === 1
|
|
478
|
+
? `asset_generate ${JSON.stringify({ action: 'submit', provider: 'meshy', task: 'image-to-3d', image: paths[0] })} (or Tripo, provider "tripo", or fal's TRELLIS.2, provider "fal")`
|
|
479
|
+
: `asset_generate ${JSON.stringify({ action: 'submit', provider: 'meshy', task: 'multi-image-to-3d', images: paths })} (or Tripo, provider "tripo")`
|
|
480
|
+
return {
|
|
481
|
+
text: [
|
|
482
|
+
`Made ${made.length} ${made.length === 1 ? 'picture' : 'pictures'} with ${provider.label} (${model}) in ${((deps.now() - started) / 1000).toFixed(1)} s: ${made.join('; ')}.`,
|
|
483
|
+
`Recorded in assets.lock.json as generated, with the prompt's hash${from.length ? ` and what ${made.length === 1 ? 'it' : 'they'} started from (${from.map((f) => f.path).join(', ')})` : ''}.`,
|
|
484
|
+
provider.paid ? `Up to ${usd(spentUsd)}, recorded in the spend.` : `On the creator's ${PRICES.images?.codex?.plan ?? 'ChatGPT'} plan: nothing recorded in the spend.`,
|
|
485
|
+
failed ? `${provider.label} failed after ${back} of the ${targets.length} views: ${failed}. Make the rest again, or go on with ${back === 1 ? 'the one' : 'those'} made.` : back < targets.length ? `Only ${back} of the ${targets.length} views came back.` : null,
|
|
486
|
+
'Look at them: the whole object, alone, nothing in front of it, no lettering, in the brief\'s form and materials. If one is wrong, make it again with words that fix it.',
|
|
487
|
+
`Next: ${next}, then asset_job until it is done, then asset_import what it made with the class and the size your task gives.`,
|
|
488
|
+
]
|
|
489
|
+
.filter(Boolean)
|
|
490
|
+
.join('\n'),
|
|
491
|
+
...(images.length ? { images } : {}),
|
|
492
|
+
}
|
|
493
|
+
}
|
|
494
|
+
|
|
495
|
+
/** A style direction's sample: written to refs/ for the style card, and
|
|
496
|
+
* shown, so the agent sees it is drawn the way the direction says. */
|
|
497
|
+
async function landDirection({ rel, picture, provider, model, spentUsd, root, seconds }) {
|
|
498
|
+
let png
|
|
499
|
+
try {
|
|
500
|
+
png = await asPng(picture)
|
|
501
|
+
} catch (err) {
|
|
502
|
+
return fail(`${provider.label}'s picture could not be read (${err?.message ?? err}); nothing was written.`)
|
|
503
|
+
}
|
|
504
|
+
await writeAtomic(path.join(root, ...rel.split('/')), png.bytes)
|
|
505
|
+
return {
|
|
506
|
+
text: [
|
|
507
|
+
`Made ${rel}${png.width ? ` (${png.width}x${png.height})` : ''} with ${provider.label} (${model}) in ${seconds.toFixed(1)} s.`,
|
|
508
|
+
provider.paid ? `Up to ${usd(spentUsd)}, recorded in the spend.` : `On the creator's ${PRICES.images?.codex?.plan ?? 'ChatGPT'} plan: nothing recorded in the spend.`,
|
|
509
|
+
'Look at it: drawn wholly in this direction\'s way, unlike the other directions and unlike the creator\'s other spaces. Make it again with words that fix it when it is not.',
|
|
510
|
+
'Next: once every direction has its sample, nw_ask_pictures with purpose "style", one picture a direction (label: its name; what: its medium and why it suits the request).',
|
|
511
|
+
]
|
|
512
|
+
.filter(Boolean)
|
|
513
|
+
.join('\n'),
|
|
514
|
+
...(png.decoded ? { images: [{ mime: 'image/png', data: encodePng(fitEdge(png.decoded, 768)).toString('base64') }] } : {}),
|
|
515
|
+
}
|
|
516
|
+
}
|
|
517
|
+
|
|
518
|
+
/** The target pictures: each written as its hero view's PNG in the round's
|
|
519
|
+
* folder, with one source.json beside them that records how each was made
|
|
520
|
+
* and its camera (nw_ask_targets and nw_round read it back). */
|
|
521
|
+
async function landTargets({ heroes, targets, pictures, prompts, provider, model, jobId, root, lines, spentUsd, failed, seconds }) {
|
|
522
|
+
const made = []
|
|
523
|
+
const images = []
|
|
524
|
+
const records = []
|
|
525
|
+
for (let k = 0; k < pictures.length; k++) {
|
|
526
|
+
let png
|
|
527
|
+
try {
|
|
528
|
+
png = await asPng(pictures[k])
|
|
529
|
+
} catch (err) {
|
|
530
|
+
return fail(`${provider.label}'s picture for ${heroes[k].id} could not be read (${err?.message ?? err}); ${made.length ? `${made.join(', ')} ${made.length === 1 ? 'was' : 'were'} written` : 'nothing was written'}.`)
|
|
531
|
+
}
|
|
532
|
+
const rel = targets[k]
|
|
533
|
+
await writeAtomic(path.join(root, ...rel.split('/')), png.bytes)
|
|
534
|
+
const h = heroes[k]
|
|
535
|
+
records.push({
|
|
536
|
+
hero: h.id,
|
|
537
|
+
path: rel,
|
|
538
|
+
view: h.view,
|
|
539
|
+
...(h.position ? { position: h.position } : {}),
|
|
540
|
+
...(h.lookAt ? { lookAt: h.lookAt } : {}),
|
|
541
|
+
...(Number.isFinite(h.fov) ? { fov: h.fov } : {}),
|
|
542
|
+
vendor: provider.id,
|
|
543
|
+
model,
|
|
544
|
+
job: `${jobId}-${h.id}`,
|
|
545
|
+
usd: provider.paid ? lines[k].usd : 0,
|
|
546
|
+
promptSha256: sha(prompts[k]),
|
|
547
|
+
// The picture's own bytes: nw_ask_targets takes a file here as a drawn
|
|
548
|
+
// target only when it is still the one drawn.
|
|
549
|
+
sha256: sha(png.bytes),
|
|
550
|
+
prompt: prompts[k].slice(0, 4000),
|
|
551
|
+
...(png.width ? { width: png.width, height: png.height } : {}),
|
|
552
|
+
})
|
|
553
|
+
made.push(`${h.id}: ${rel}${png.width ? ` (${png.width}x${png.height})` : ''}`)
|
|
554
|
+
if (png.decoded) images.push({ mime: 'image/png', data: encodePng(fitEdge(png.decoded, 768)).toString('base64') })
|
|
555
|
+
}
|
|
556
|
+
const dir = path.dirname(path.join(root, ...targets[0].split('/')))
|
|
557
|
+
await writeAtomic(path.join(dir, 'source.json'), Buffer.from(`${JSON.stringify({ schema: 'nowhere/targets@1', made: new Date().toISOString(), targets: records }, null, 2)}\n`))
|
|
558
|
+
const pairs = records.map((r) => ({ path: r.path, hero: r.hero }))
|
|
559
|
+
return {
|
|
560
|
+
text: [
|
|
561
|
+
`Made ${made.length} target ${made.length === 1 ? 'picture' : 'pictures'} with ${provider.label} (${model}) in ${seconds.toFixed(1)} s: ${made.join('; ')}. Their record (camera, vendor, model, the prompt and its hash) is ${path.relative(root, dir).split(path.sep).join('/')}/source.json; they are not the space's files.`,
|
|
562
|
+
provider.paid ? `Up to ${usd(spentUsd)}, recorded in the spend.` : `On the creator's ${PRICES.images?.codex?.plan ?? 'ChatGPT'} plan: nothing recorded in the spend.`,
|
|
563
|
+
failed ? `${provider.label} failed after ${pictures.length} of the ${targets.length}: ${failed}. Make the rest again, or go on with those made.` : pictures.length < targets.length ? `Only ${pictures.length} of the ${targets.length} came back.` : null,
|
|
564
|
+
'Look at them: each the finished place from its camera, drawn in the space\'s style (style.json) and every one the same way, with none of the slop list in it. Make one again with words that fix it when it is wrong.',
|
|
565
|
+
`Next: nw_ask_targets with images ${JSON.stringify(pairs)} (each with what it shows), so the creator keeps, drops or sends them back with notes.`,
|
|
566
|
+
]
|
|
567
|
+
.filter(Boolean)
|
|
568
|
+
.join('\n'),
|
|
569
|
+
...(images.length ? { images } : {}),
|
|
570
|
+
}
|
|
571
|
+
}
|
|
572
|
+
|
|
573
|
+
/** A texture, panorama or card: shaped and measured, then into quarantine
|
|
574
|
+
* with its source.json for asset_import. */
|
|
575
|
+
async function landKind({ kind, name, provider, model, prompt, picture, jobId, from, root, ctx, usd: cost, spentUsd, seconds }) {
|
|
576
|
+
let png
|
|
577
|
+
try {
|
|
578
|
+
png = await asPng(picture)
|
|
579
|
+
} catch (err) {
|
|
580
|
+
return fail(`${provider.label}'s picture could not be read (${err?.message ?? err}); nothing was written.`)
|
|
581
|
+
}
|
|
582
|
+
if (!png.decoded) return fail(`${provider.label}'s picture could not be decoded here (no image decoder), so it could not be made into a ${kind}; nothing was written.`)
|
|
583
|
+
const shaped = shapeKind(kind, png.decoded, name)
|
|
584
|
+
const qid = `img-${jobId}`
|
|
585
|
+
const dir = path.join(root, '.nowhere', 'quarantine', qid)
|
|
586
|
+
await fsp.mkdir(dir, { recursive: true })
|
|
587
|
+
const bytes = encodePng(shaped.pic)
|
|
588
|
+
await writeAtomic(path.join(dir, shaped.file), bytes)
|
|
589
|
+
const source = {
|
|
590
|
+
provider: provider.id,
|
|
591
|
+
id: jobId,
|
|
592
|
+
kind: kind === 'texture' ? 'material' : 'texture',
|
|
593
|
+
title: prompt.split('\n')[0].slice(0, 200),
|
|
594
|
+
author: `${provider.label} (generated)`,
|
|
595
|
+
license: provider.licenseOf(model),
|
|
596
|
+
fetchedAt: today(),
|
|
597
|
+
job: jobId,
|
|
598
|
+
generation: { vendor: provider.id, model, prompt: prompt.slice(0, 2000), job: jobId, usd: cost },
|
|
599
|
+
provenance: { promptSha256: sha(prompt), inputs: from },
|
|
600
|
+
files: [{ path: shaped.file, bytes: bytes.length, ...(kind === 'texture' ? { role: 'color' } : {}) }],
|
|
601
|
+
meta: { ...shaped.meta, width: shaped.pic.width, height: shaped.pic.height },
|
|
602
|
+
}
|
|
603
|
+
await fsp.writeFile(path.join(dir, 'source.json'), `${JSON.stringify(source, null, 2)}\n`)
|
|
604
|
+
await addToManifest(root, { id: qid, file: qid, name: `${provider.label} ${kind} ${name}`, url: '', referrer: '', host: '', bytes: bytes.length, sha256: '', kind: source.kind, ok: true, problem: null, at: new Date().toISOString(), source: { provider: provider.id, job: jobId, license: source.license } })
|
|
605
|
+
const images = [{ mime: 'image/png', data: encodePng(fitEdge(shaped.pic, 768)).toString('base64') }]
|
|
606
|
+
if (shaped.sheet) images.push({ mime: 'image/png', data: encodePng(shaped.sheet).toString('base64') })
|
|
607
|
+
const next =
|
|
608
|
+
kind === 'texture'
|
|
609
|
+
? `asset_import ${JSON.stringify({ from: qid, as: `assets/${name}` })} as a material (its colour map; add tileSize, the metres one tile covers in the brief), or as "assets/${name}.png" for the picture alone`
|
|
610
|
+
: kind === 'panorama'
|
|
611
|
+
? `asset_import ${JSON.stringify({ from: qid, as: `assets/${name}.hdr` })}, which writes it as the Radiance file the config's hdri sky reads and answers the sky value for look_apply`
|
|
612
|
+
: `asset_import ${JSON.stringify({ from: qid, as: `assets/${name}.png` })}`
|
|
613
|
+
return {
|
|
614
|
+
text: [
|
|
615
|
+
`Made a ${kind} with ${provider.label} (${model}) in ${seconds.toFixed(1)} s: ${shaped.file}, ${shaped.pic.width}x${shaped.pic.height}, in quarantine as ${qid} with its source.json (the vendor, the model, the prompt and its hash${from.length ? ', what it started from' : ''}).`,
|
|
616
|
+
...shaped.lines,
|
|
617
|
+
provider.paid ? `Up to ${usd(spentUsd)}, recorded in the spend.` : `On the creator's ${PRICES.images?.codex?.plan ?? 'ChatGPT'} plan: nothing recorded in the spend.`,
|
|
618
|
+
shaped.after?.verdict === 'seam' ? 'It still does not tile cleanly: make it again with words for an even, edge-to-edge surface, or keep it only where its edges are hidden.' : null,
|
|
619
|
+
`Look at it${shaped.sheet ? ' (the second picture is it tiled 2 x 2: a seam shows as a cross through the middle)' : ''}, then ${next}.`,
|
|
620
|
+
]
|
|
621
|
+
.filter(Boolean)
|
|
622
|
+
.join('\n'),
|
|
623
|
+
images,
|
|
624
|
+
}
|
|
625
|
+
}
|
|
626
|
+
|
|
627
|
+
// ---------------------------------------------------------------- the defs
|
|
628
|
+
|
|
629
|
+
/** The ToolDef, with Codex's search, its machine and the clock replaceable for checks. */
|
|
630
|
+
export function makeImageTools(overrides = {}) {
|
|
631
|
+
const deps = { codexDeps, now: () => Date.now(), ...overrides }
|
|
632
|
+
deps.findCodex ??= () => findCodex(deps.codexDeps)
|
|
633
|
+
const guarded = async (args, ctx) => {
|
|
634
|
+
try {
|
|
635
|
+
return await generate(args, ctx, deps)
|
|
636
|
+
} catch (err) {
|
|
637
|
+
return fail(`image_generate failed: ${err?.message ?? err}`)
|
|
638
|
+
}
|
|
639
|
+
}
|
|
640
|
+
return [{ name: 'image_generate', description: DESCRIPTION, shape: SHAPE, readOnly: false, handler: guarded }]
|
|
641
|
+
}
|
|
642
|
+
|
|
643
|
+
export const TOOLS = makeImageTools()
|