magpie-gemini-web 0.6.0 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.en.md +43 -9
- package/README.md +43 -9
- package/docs/logo.svg +16 -0
- package/docs/studio-release.md +99 -20
- package/index.mjs +2 -1
- package/lib/backend.mjs +121 -3
- package/lib/studio-native.mjs +7 -3
- package/lib/studio-release.json +73 -19
- package/lib/studio.mjs +82 -12
- package/package.json +7 -2
- package/patches/studio-0.2.17-preserve-request-bodies.patch +160 -0
- package/scripts/studio-managed-smoke.mjs +25 -1
- package/scripts/studio-pins.mjs +49 -0
- package/scripts/studio-release.mjs +193 -18
|
@@ -0,0 +1,160 @@
|
|
|
1
|
+
diff --git a/internal/requestdb/legacy_bodies.go b/internal/requestdb/legacy_bodies.go
|
|
2
|
+
new file mode 100644
|
|
3
|
+
index 0000000..5569c03
|
|
4
|
+
--- /dev/null
|
|
5
|
+
+++ b/internal/requestdb/legacy_bodies.go
|
|
6
|
+
@@ -0,0 +1,49 @@
|
|
7
|
+
+package requestdb
|
|
8
|
+
+
|
|
9
|
+
+import "database/sql"
|
|
10
|
+
+
|
|
11
|
+
+// migrateLegacyBodies preserves pre-0.2.17 plaintext bodies before removing their old table.
|
|
12
|
+
+func migrateLegacyBodies(db *sql.DB) error {
|
|
13
|
+
+ var exists bool
|
|
14
|
+
+ if err := db.QueryRow(`SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE type='table' AND name='bodies')`).Scan(&exists); err != nil || !exists {
|
|
15
|
+
+ return err
|
|
16
|
+
+ }
|
|
17
|
+
+ tx, err := db.Begin()
|
|
18
|
+
+ if err != nil {
|
|
19
|
+
+ return err
|
|
20
|
+
+ }
|
|
21
|
+
+ defer tx.Rollback()
|
|
22
|
+
+ rows, err := tx.Query(`SELECT id,time,request,request_size,response,response_size FROM bodies ORDER BY seq`)
|
|
23
|
+
+ if err != nil {
|
|
24
|
+
+ return err
|
|
25
|
+
+ }
|
|
26
|
+
+ defer rows.Close()
|
|
27
|
+
+ for rows.Next() {
|
|
28
|
+
+ var id, request, response string
|
|
29
|
+
+ var at, requestSize, responseSize int64
|
|
30
|
+
+ if err := rows.Scan(&id, &at, &request, &requestSize, &response, &responseSize); err != nil {
|
|
31
|
+
+ return err
|
|
32
|
+
+ }
|
|
33
|
+
+ req, err := compress(request)
|
|
34
|
+
+ if err != nil {
|
|
35
|
+
+ return err
|
|
36
|
+
+ }
|
|
37
|
+
+ res, err := compress(response)
|
|
38
|
+
+ if err != nil {
|
|
39
|
+
+ return err
|
|
40
|
+
+ }
|
|
41
|
+
+ if _, err := tx.Exec(`INSERT OR IGNORE INTO request_bodies (id,time,request,request_size,response,response_size) VALUES (?,?,?,?,?,?)`, id, at, req, requestSize, res, responseSize); err != nil {
|
|
42
|
+
+ return err
|
|
43
|
+
+ }
|
|
44
|
+
+ }
|
|
45
|
+
+ if err := rows.Err(); err != nil {
|
|
46
|
+
+ return err
|
|
47
|
+
+ }
|
|
48
|
+
+ if err := rows.Close(); err != nil {
|
|
49
|
+
+ return err
|
|
50
|
+
+ }
|
|
51
|
+
+ if _, err := tx.Exec(`DROP TABLE bodies`); err != nil {
|
|
52
|
+
+ return err
|
|
53
|
+
+ }
|
|
54
|
+
+ return tx.Commit()
|
|
55
|
+
+}
|
|
56
|
+
diff --git a/internal/requestdb/legacy_bodies_test.go b/internal/requestdb/legacy_bodies_test.go
|
|
57
|
+
new file mode 100644
|
|
58
|
+
index 0000000..2761c06
|
|
59
|
+
--- /dev/null
|
|
60
|
+
+++ b/internal/requestdb/legacy_bodies_test.go
|
|
61
|
+
@@ -0,0 +1,77 @@
|
|
62
|
+
+package requestdb
|
|
63
|
+
+
|
|
64
|
+
+import (
|
|
65
|
+
+ "context"
|
|
66
|
+
+ "path/filepath"
|
|
67
|
+
+ "testing"
|
|
68
|
+
+
|
|
69
|
+
+ "github.com/Mag1cFall/AIStudio2API/internal/sqlitedb"
|
|
70
|
+
+)
|
|
71
|
+
+
|
|
72
|
+
+func TestLegacyBodiesMigration(t *testing.T) {
|
|
73
|
+
+ file := filepath.Join(t.TempDir(), "requests.db")
|
|
74
|
+
+ db, err := sqlitedb.Open(file)
|
|
75
|
+
+ if err != nil {
|
|
76
|
+
+ t.Fatal(err)
|
|
77
|
+
+ }
|
|
78
|
+
+ _, err = db.Exec(`CREATE TABLE bodies (seq INTEGER PRIMARY KEY AUTOINCREMENT, id TEXT NOT NULL UNIQUE, time INTEGER NOT NULL, request TEXT NOT NULL, request_size INTEGER NOT NULL, response TEXT NOT NULL, response_size INTEGER NOT NULL);
|
|
79
|
+
+INSERT INTO bodies (id,time,request,request_size,response,response_size) VALUES ('old',1234,'旧请求',100,'reply',200);`)
|
|
80
|
+
+ if err != nil {
|
|
81
|
+
+ t.Fatal(err)
|
|
82
|
+
+ }
|
|
83
|
+
+ if err := db.Close(); err != nil {
|
|
84
|
+
+ t.Fatal(err)
|
|
85
|
+
+ }
|
|
86
|
+
+ for round := range 2 {
|
|
87
|
+
+ store, err := Open(file, func(string, string) {})
|
|
88
|
+
+ if err != nil {
|
|
89
|
+
+ t.Fatal(err)
|
|
90
|
+
+ }
|
|
91
|
+
+ body, err := store.RequestBody(context.Background(), "old")
|
|
92
|
+
+ if err != nil {
|
|
93
|
+
+ t.Fatal(err)
|
|
94
|
+
+ }
|
|
95
|
+
+ if body.Request != "旧请求" || body.Response != "reply" || body.RequestSize != 100 || body.ResponseSize != 200 || body.Time.UnixMilli() != 1234 {
|
|
96
|
+
+ t.Fatalf("changed body: %+v", body)
|
|
97
|
+
+ }
|
|
98
|
+
+ var old int
|
|
99
|
+
+ if err := store.db.QueryRow(`SELECT COUNT(*) FROM sqlite_master WHERE name='bodies'`).Scan(&old); err != nil || old != 0 {
|
|
100
|
+
+ t.Fatalf("old table: %d, %v", old, err)
|
|
101
|
+
+ }
|
|
102
|
+
+ if round == 0 {
|
|
103
|
+
+ // A retry must keep an already migrated body over a stale legacy copy.
|
|
104
|
+
+ if _, err := store.db.Exec(`CREATE TABLE bodies (seq INTEGER, id TEXT, time INTEGER, request TEXT, request_size INTEGER, response TEXT, response_size INTEGER);
|
|
105
|
+
+INSERT INTO bodies VALUES (1,'old',9999,'stale',1,'stale',1);`); err != nil {
|
|
106
|
+
+ t.Fatal(err)
|
|
107
|
+
+ }
|
|
108
|
+
+ }
|
|
109
|
+
+ if err := store.Close(); err != nil {
|
|
110
|
+
+ t.Fatal(err)
|
|
111
|
+
+ }
|
|
112
|
+
+ }
|
|
113
|
+
+}
|
|
114
|
+
+
|
|
115
|
+
+func TestLegacyBodiesMigrationRollback(t *testing.T) {
|
|
116
|
+
+ db, err := sqlitedb.Open(filepath.Join(t.TempDir(), "requests.db"))
|
|
117
|
+
+ if err != nil {
|
|
118
|
+
+ t.Fatal(err)
|
|
119
|
+
+ }
|
|
120
|
+
+ defer db.Close()
|
|
121
|
+
+ if _, err := db.Exec(schema + `CREATE TABLE bodies (seq INTEGER, id TEXT, time INTEGER, request TEXT, request_size INTEGER, response TEXT, response_size INTEGER);
|
|
122
|
+
+INSERT INTO bodies VALUES (1,'good',1234,'request',1,'reply',1),(2,'bad',1234,NULL,1,'reply',1);`); err != nil {
|
|
123
|
+
+ t.Fatal(err)
|
|
124
|
+
+ }
|
|
125
|
+
+ if err := migrateLegacyBodies(db); err == nil {
|
|
126
|
+
+ t.Fatal("expected invalid legacy row to fail")
|
|
127
|
+
+ }
|
|
128
|
+
+ var old, migrated int
|
|
129
|
+
+ if err := db.QueryRow(`SELECT COUNT(*) FROM bodies`).Scan(&old); err != nil {
|
|
130
|
+
+ t.Fatal(err)
|
|
131
|
+
+ }
|
|
132
|
+
+ if err := db.QueryRow(`SELECT COUNT(*) FROM request_bodies`).Scan(&migrated); err != nil {
|
|
133
|
+
+ t.Fatal(err)
|
|
134
|
+
+ }
|
|
135
|
+
+ if old != 2 || migrated != 0 {
|
|
136
|
+
+ t.Fatalf("migration was not atomic: old=%d new=%d", old, migrated)
|
|
137
|
+
+ }
|
|
138
|
+
+}
|
|
139
|
+
diff --git a/internal/requestdb/requestdb.go b/internal/requestdb/requestdb.go
|
|
140
|
+
index bff3477..ae92661 100644
|
|
141
|
+
--- a/internal/requestdb/requestdb.go
|
|
142
|
+
+++ b/internal/requestdb/requestdb.go
|
|
143
|
+
@@ -58,7 +58,6 @@ CREATE INDEX IF NOT EXISTS requests_account_time ON requests(account, time);
|
|
144
|
+
CREATE INDEX IF NOT EXISTS requests_state_time ON requests(state, time);
|
|
145
|
+
CREATE TABLE IF NOT EXISTS hours (` + rollupColumns + `) WITHOUT ROWID;
|
|
146
|
+
CREATE TABLE IF NOT EXISTS days (` + rollupColumns + `) WITHOUT ROWID;
|
|
147
|
+
-DROP TABLE IF EXISTS bodies;
|
|
148
|
+
CREATE TABLE IF NOT EXISTS request_bodies (
|
|
149
|
+
seq INTEGER PRIMARY KEY AUTOINCREMENT, id TEXT NOT NULL UNIQUE, time INTEGER NOT NULL,
|
|
150
|
+
request BLOB NOT NULL, request_size INTEGER NOT NULL, response BLOB NOT NULL, response_size INTEGER NOT NULL);`
|
|
151
|
+
@@ -121,6 +120,9 @@ func Open(path string, log Logger) (*Store, error) {
|
|
152
|
+
if _, err := db.Exec(schema); err != nil {
|
|
153
|
+
return nil, errors.Join(fmt.Errorf("初始化请求账本: %w", err), db.Close())
|
|
154
|
+
}
|
|
155
|
+
+ if err := migrateLegacyBodies(db); err != nil {
|
|
156
|
+
+ return nil, errors.Join(fmt.Errorf("迁移请求正文: %w", err), db.Close())
|
|
157
|
+
+ }
|
|
158
|
+
if err := addCachedTokens(db); err != nil {
|
|
159
|
+
return nil, errors.Join(fmt.Errorf("初始化请求账本: %w", err), db.Close())
|
|
160
|
+
}
|
|
@@ -4,6 +4,7 @@ import fs from 'node:fs';
|
|
|
4
4
|
import os from 'node:os';
|
|
5
5
|
import path from 'node:path';
|
|
6
6
|
import { fileURLToPath, pathToFileURL } from 'node:url';
|
|
7
|
+
import { DatabaseSync } from 'node:sqlite';
|
|
7
8
|
|
|
8
9
|
const root = fileURLToPath(new URL('..', import.meta.url));
|
|
9
10
|
const args = process.argv.slice(2);
|
|
@@ -22,7 +23,27 @@ try {
|
|
|
22
23
|
local = await installStudio({ dataDir, ...(archivePath ? { archivePath } : {}) });
|
|
23
24
|
const bin = path.join(dataDir, 'package', process.platform === 'win32' ? 'aistudio2api.exe' : 'aistudio2api');
|
|
24
25
|
assert.equal(computeFileHash(bin), pin.binarySha256);
|
|
26
|
+
// A pre-0.2.17 ledger must survive the real process's first startup.
|
|
27
|
+
fs.mkdirSync(path.join(dataDir, 'runtime'), { recursive: true });
|
|
28
|
+
const db = new DatabaseSync(path.join(dataDir, 'runtime', 'requests.db'));
|
|
29
|
+
try {
|
|
30
|
+
db.exec(`CREATE TABLE bodies (seq INTEGER PRIMARY KEY AUTOINCREMENT, id TEXT NOT NULL UNIQUE, time INTEGER NOT NULL,
|
|
31
|
+
request TEXT NOT NULL, request_size INTEGER NOT NULL, response TEXT NOT NULL, response_size INTEGER NOT NULL)`);
|
|
32
|
+
db.prepare('INSERT INTO bodies (id,time,request,request_size,response,response_size) VALUES (?,?,?,?,?,?)')
|
|
33
|
+
.run('legacy-smoke', 1234, '旧请求', 100, 'old response', 200);
|
|
34
|
+
} finally { db.close(); }
|
|
35
|
+
fs.appendFileSync(path.join(dataDir, '.env'), 'FIRST_EVENT_HEDGE=auto\n');
|
|
25
36
|
await ensureStudio({ dataDir, managementOnly: true });
|
|
37
|
+
const migratedBody = await studioAdmin(local, 'GET', '/api/requests/legacy-smoke/body');
|
|
38
|
+
assert.equal(migratedBody.request, '旧请求');
|
|
39
|
+
assert.equal(migratedBody.response, 'old response');
|
|
40
|
+
assert.equal(migratedBody.request_size, 100);
|
|
41
|
+
assert.equal(migratedBody.response_size, 200);
|
|
42
|
+
const config = await studioAdmin(local, 'GET', '/api/config');
|
|
43
|
+
assert.equal(config.first_event_hedge, 'auto');
|
|
44
|
+
// Generation's WAA runs in the service itself, on SpiderMonkey, with no copies of the models
|
|
45
|
+
assert.equal(config.waa_backend, 'spidermonkey');
|
|
46
|
+
assert.equal(config.search_model_aliases, false);
|
|
26
47
|
const pidBefore = fs.readFileSync(path.join(dataDir, 'backend.pid'), 'utf8');
|
|
27
48
|
const status = await studioAdmin(local, 'GET', '/api/status');
|
|
28
49
|
assert.equal(status.version, STUDIO_VERSION);
|
|
@@ -57,7 +78,10 @@ try {
|
|
|
57
78
|
assert.match(panel, /version\.managed/);
|
|
58
79
|
assert.deepEqual(await stopStudio({ dataDir }), { stopped: true });
|
|
59
80
|
assert.equal((await studioStatus({ dataDir })).status, 'stopped');
|
|
60
|
-
|
|
81
|
+
await ensureStudio({ dataDir, managementOnly: true });
|
|
82
|
+
assert.deepEqual(await studioAdmin(local, 'GET', '/api/requests/legacy-smoke/body'), migratedBody);
|
|
83
|
+
assert.deepEqual(await stopStudio({ dataDir }), { stopped: true });
|
|
84
|
+
console.log(`PASS ${key}: ${STUDIO_VERSION} (${source}), legacy body migration/restart, saved hedge setting, SpiderMonkey WAA, managed status, update HTTP 409, unchanged executable/PID, managed UI, clean stop`);
|
|
61
85
|
} finally {
|
|
62
86
|
let stopped = !local;
|
|
63
87
|
if (local) stopped = (await stopStudio({ dataDir }).catch(() => ({ stopped: false }))).stopped;
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
// Candidate build records -> lib/studio-release.json binaries. A candidate run of
|
|
3
|
+
// .github/workflows/studio.yml builds each target on its own runner with --write-pins; this takes
|
|
4
|
+
// the build-<target>.json files those runners upload and pins all five targets at once.
|
|
5
|
+
// Usage: node scripts/studio-pins.mjs candidates/*/build-*.json
|
|
6
|
+
import fs from 'node:fs';
|
|
7
|
+
import path from 'node:path';
|
|
8
|
+
import crypto from 'node:crypto';
|
|
9
|
+
import assert from 'node:assert/strict';
|
|
10
|
+
import { fileURLToPath } from 'node:url';
|
|
11
|
+
|
|
12
|
+
const root = fileURLToPath(new URL('..', import.meta.url));
|
|
13
|
+
const pinPath = path.join(root, 'lib/studio-release.json');
|
|
14
|
+
|
|
15
|
+
// mergePins returns release with the binaries the records built. Each record must come from this
|
|
16
|
+
// release's source, patches and toolchains, and each target from exactly one record.
|
|
17
|
+
export function mergePins(release, records, patchHashes) {
|
|
18
|
+
const patches = release.patches.map(name => ({ name, sha256: patchHashes[name] }));
|
|
19
|
+
const built = {};
|
|
20
|
+
for (const { file, record } of records) {
|
|
21
|
+
assert.equal(record.version, release.version, `${file}: built version ${record.version}`);
|
|
22
|
+
assert.deepEqual(record.upstream, release.upstream, `${file}: built from another upstream`);
|
|
23
|
+
assert.equal(record.goVersion, release.goVersion, `${file}: built with Go ${record.goVersion}`);
|
|
24
|
+
assert.deepEqual(record.toolchains, release.toolchains, `${file}: built with other pinned toolchains`);
|
|
25
|
+
assert.deepEqual(record.patches, patches, `${file}: built with other patches`);
|
|
26
|
+
for (const [key, pin] of Object.entries(record.binaries)) {
|
|
27
|
+
assert.ok(key in release.binaries, `${file}: unknown target ${key}`);
|
|
28
|
+
assert.ok(!(key in built), `${key} is in more than one build record`);
|
|
29
|
+
assert.match(pin.asset, new RegExp(`^aistudio2api-v${release.version.replaceAll('.', '\\.')}-`), `${file}: ${key} asset ${pin.asset}`);
|
|
30
|
+
for (const field of ['sha256', 'binarySha256']) assert.match(pin[field], /^[0-9a-f]{64}$/, `${file}: ${key} ${field}`);
|
|
31
|
+
built[key] = { asset: pin.asset, sha256: pin.sha256, binarySha256: pin.binarySha256 };
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
const missing = Object.keys(release.binaries).filter(key => !(key in built));
|
|
35
|
+
assert.deepEqual(missing, [], `No build record for ${missing.join(', ')}`);
|
|
36
|
+
return { ...release, binaries: Object.fromEntries(Object.keys(release.binaries).map(key => [key, built[key]])) };
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) {
|
|
40
|
+
const files = process.argv.slice(2);
|
|
41
|
+
if (!files.length) throw new Error('Usage: node scripts/studio-pins.mjs <build-record.json>...');
|
|
42
|
+
const release = JSON.parse(fs.readFileSync(pinPath, 'utf8'));
|
|
43
|
+
const patchHashes = Object.fromEntries(release.patches.map(name => [name,
|
|
44
|
+
crypto.createHash('sha256').update(fs.readFileSync(path.join(root, 'patches', name))).digest('hex')]));
|
|
45
|
+
const records = files.map(file => ({ file, record: JSON.parse(fs.readFileSync(file, 'utf8')) }));
|
|
46
|
+
const merged = mergePins(release, records, patchHashes);
|
|
47
|
+
fs.writeFileSync(pinPath, JSON.stringify(merged, null, 2) + '\n');
|
|
48
|
+
for (const [key, pin] of Object.entries(merged.binaries)) console.log(`${key}: ${pin.sha256} ${pin.asset}`);
|
|
49
|
+
}
|
|
@@ -18,29 +18,188 @@ for (const arg of args) {
|
|
|
18
18
|
}
|
|
19
19
|
const writePins = args.includes('--write-pins');
|
|
20
20
|
const verify = args.includes('--verify');
|
|
21
|
-
|
|
22
|
-
const
|
|
21
|
+
// Pins are only as good as the toolchain that made them: checking or writing them needs the pinned one.
|
|
22
|
+
const strict = writePins || verify;
|
|
23
23
|
const platforms = { 'darwin-arm64': ['darwin', 'arm64'], 'darwin-x64': ['darwin', 'amd64'],
|
|
24
24
|
'linux-x64': ['linux', 'amd64'], 'linux-arm64': ['linux', 'arm64'], 'win32-x64': ['windows', 'amd64'] };
|
|
25
|
-
|
|
26
|
-
|
|
25
|
+
// --target=host, one target, or a comma-separated list; all targets without it.
|
|
26
|
+
const requested = args.find(arg => arg.startsWith('--target='))?.slice(9);
|
|
27
|
+
const targets = requested ? requested.split(',').map(key => key === 'host' ? `${process.platform}-${process.arch}` : key)
|
|
28
|
+
: Object.keys(platforms);
|
|
29
|
+
for (const key of targets) if (!platforms[key]) throw new Error(`Unsupported target: ${key}`);
|
|
30
|
+
if (new Set(targets).size !== targets.length) throw new Error(`Duplicate target in --target=${requested}`);
|
|
31
|
+
if (writePins && verify) throw new Error('--write-pins cannot be combined with --verify');
|
|
27
32
|
const output = path.join(root, 'studio-build/release');
|
|
28
33
|
fs.mkdirSync(output, { recursive: true });
|
|
29
34
|
const scratch = fs.mkdtempSync(path.join(os.tmpdir(), 'magpie-studio-build-'));
|
|
30
35
|
const source = path.join(scratch, 'source');
|
|
31
36
|
fs.mkdirSync(source);
|
|
32
|
-
const env = { ...process.env, CGO_ENABLED: '
|
|
37
|
+
const env = { ...process.env, CGO_ENABLED: '1', GOFLAGS: '', GOEXPERIMENT: '',
|
|
33
38
|
GOTOOLCHAIN: `go${release.goVersion}`, GOAMD64: 'v1', GOARM64: 'v8.0' };
|
|
34
|
-
for (const key of ['GOOS', 'GOARCH']) delete env[key];
|
|
39
|
+
for (const key of ['GOOS', 'GOARCH', 'CC', 'CXX', 'CGO_CFLAGS', 'CGO_CPPFLAGS', 'CGO_CXXFLAGS', 'CGO_LDFLAGS', 'MACOSX_DEPLOYMENT_TARGET']) delete env[key];
|
|
35
40
|
const hash = file => crypto.createHash('sha256').update(fs.readFileSync(file)).digest('hex');
|
|
41
|
+
// SQLite needs C and SpiderMonkey C++, so the C/C++ toolchain is part of the pin (lib/studio-release.json
|
|
42
|
+
// "toolchains"). Zig (Linux/musl) and llvm-mingw (Windows/UCRT) are cross compilers whose output does
|
|
43
|
+
// not depend on the machine they run on, so any builder reproduces those archives. Apple's linker and
|
|
44
|
+
// SDK are only on macOS and differ between Xcode releases, so macOS archives match only under the
|
|
45
|
+
// pinned Xcode. SpiderMonkey's static libraries are upstream's prebuilt ones, pinned per target.
|
|
46
|
+
const toolchains = release.toolchains;
|
|
47
|
+
const zig = process.env.STUDIO_ZIG || 'zig';
|
|
48
|
+
function pinned(what, actual, expected, hint) {
|
|
49
|
+
if (actual === expected) return;
|
|
50
|
+
const message = `${what} is ${actual || 'unknown'}, but the pins were made with ${expected}. ${hint}`;
|
|
51
|
+
if (strict) throw new Error(message);
|
|
52
|
+
console.warn(`warning: ${message}`);
|
|
53
|
+
}
|
|
54
|
+
// fetchPinned returns studio-build/toolchains/<name>, where the archive at url is unpacked once after
|
|
55
|
+
// its SHA-256 is checked against the pinned one. top is the folder the archive holds everything in,
|
|
56
|
+
// or null for an archive that unpacks in place; prepare, if given, changes what was unpacked first.
|
|
57
|
+
function fetchPinned(what, url, { asset, sha256 }, name, top, prepare) {
|
|
58
|
+
const cacheDir = path.join(root, 'studio-build/toolchains');
|
|
59
|
+
const home = path.join(cacheDir, name);
|
|
60
|
+
const stamp = path.join(home, '.sha256');
|
|
61
|
+
if (fs.existsSync(stamp) && fs.readFileSync(stamp, 'utf8').trim() === sha256) return home;
|
|
62
|
+
fs.rmSync(home, { recursive: true, force: true });
|
|
63
|
+
fs.mkdirSync(cacheDir, { recursive: true });
|
|
64
|
+
const staging = fs.mkdtempSync(path.join(cacheDir, '.fetch-'));
|
|
65
|
+
try {
|
|
66
|
+
const file = path.join(staging, asset);
|
|
67
|
+
console.log(`Fetching pinned ${what}: ${url}`);
|
|
68
|
+
execFileSync('curl', ['--fail', '--location', '--silent', '--show-error', '--retry', '3', '--output', file, url], { stdio: 'inherit' });
|
|
69
|
+
const actual = hash(file);
|
|
70
|
+
if (actual !== sha256) throw new Error(`${asset} has SHA-256 ${actual}, not the pinned ${sha256}`);
|
|
71
|
+
const unpacked = path.join(staging, top ?? 'unpacked');
|
|
72
|
+
if (!top) fs.mkdirSync(unpacked);
|
|
73
|
+
// Windows' own bsdtar reads zip; the GNU tar Git Bash puts ahead of it on PATH does not.
|
|
74
|
+
const tar = process.platform === 'win32' ? path.join(process.env.SystemRoot || 'C:\\Windows', 'System32', 'tar.exe') : 'tar';
|
|
75
|
+
execFileSync(tar, ['-xf', file, '-C', top ? staging : unpacked], { stdio: 'inherit' });
|
|
76
|
+
prepare?.(unpacked);
|
|
77
|
+
fs.writeFileSync(path.join(unpacked, '.sha256'), `${sha256}\n`);
|
|
78
|
+
fs.renameSync(unpacked, home);
|
|
79
|
+
} finally {
|
|
80
|
+
fs.rmSync(staging, { recursive: true, force: true });
|
|
81
|
+
}
|
|
82
|
+
return home;
|
|
83
|
+
}
|
|
84
|
+
// mingwHome returns the pinned llvm-mingw for a builder host, fetched into studio-build/toolchains.
|
|
85
|
+
function mingwHome(host) {
|
|
86
|
+
const pin = toolchains.llvmMingw;
|
|
87
|
+
const archive = pin.hosts[host];
|
|
88
|
+
if (!archive) throw new Error(`No pinned llvm-mingw for ${host}: set STUDIO_MINGW_CC to llvm-mingw ${pin.version}'s x86_64-w64-mingw32-clang`);
|
|
89
|
+
const name = archive.asset.replace(/\.(tar\.xz|zip)$/, '');
|
|
90
|
+
return fetchPinned('llvm-mingw', `https://github.com/mstorsjo/llvm-mingw/releases/download/${pin.version}/${archive.asset}`,
|
|
91
|
+
archive, name, name);
|
|
92
|
+
}
|
|
93
|
+
// llvmMingw returns the pinned llvm-mingw's x86_64 C and C++ compilers, unless STUDIO_MINGW_CC names
|
|
94
|
+
// a C compiler, whose C++ one is next to it. libs is where Windows' libraries are linked from first:
|
|
95
|
+
// llvm-mingw's Linux and Windows archives have the same ones, byte for byte, but its macOS archive's
|
|
96
|
+
// are built apart, and the C++ runtime among them puts its own build paths in the executable. So a
|
|
97
|
+
// macOS builder links the Linux archive's, and makes the same executable as the others. Every host
|
|
98
|
+
// names its folder all the same: cgo writes a line for each linker flag into the Go it generates,
|
|
99
|
+
// so one flag more moves the line numbers the executable keeps.
|
|
100
|
+
function llvmMingw() {
|
|
101
|
+
const exe = process.platform === 'win32' ? '.exe' : '';
|
|
102
|
+
const cxxOf = cc => cc.replace(/clang(\.exe)?$/i, 'clang++$1');
|
|
103
|
+
const libsOf = home => path.join(home, 'x86_64-w64-mingw32', 'lib');
|
|
104
|
+
const macOS = process.platform === 'darwin';
|
|
105
|
+
if (process.env.STUDIO_MINGW_CC) {
|
|
106
|
+
const cc = process.env.STUDIO_MINGW_CC;
|
|
107
|
+
return { cc, cxx: cxxOf(cc), libs: libsOf(macOS ? mingwHome('linux-x64') : path.resolve(cc, '..', '..')) };
|
|
108
|
+
}
|
|
109
|
+
const home = mingwHome(macOS ? 'darwin' : `${process.platform}-${process.arch}`);
|
|
110
|
+
const cc = path.join(home, 'bin', `x86_64-w64-mingw32-clang${exe}`);
|
|
111
|
+
return { cc, cxx: cxxOf(cc), libs: libsOf(macOS ? mingwHome('linux-x64') : home) };
|
|
112
|
+
}
|
|
113
|
+
// The Linux (musl) SpiderMonkey libraries keep mozglue's interposers for the environment's functions.
|
|
114
|
+
// But for getenv, each calls the C library's own, found with dlsym(RTLD_NEXT), and a static executable
|
|
115
|
+
// has none to find: setenv and the rest crash on purpose (MOZ_CRASH), as Go's os.Setenv does through
|
|
116
|
+
// cgo. Without them, musl's own are linked; dynamic builds, which upstream tests, find those anyway.
|
|
117
|
+
const ENV_INTERPOSERS = 'Unified_cpp_mozglue_interposers0.o';
|
|
118
|
+
function dropEnvInterposers(dir) {
|
|
119
|
+
const archive = path.join(dir, 'lib', 'libjs_static.a');
|
|
120
|
+
const members = execFileSync(zig, ['ar', 't', archive], { encoding: 'utf8' }).split('\n');
|
|
121
|
+
if (!members.includes(ENV_INTERPOSERS)) {
|
|
122
|
+
throw new Error(`${archive} has no ${ENV_INTERPOSERS}: check how this SpiderMonkey reaches the environment`);
|
|
123
|
+
}
|
|
124
|
+
execFileSync(zig, ['ar', '--format=gnu', 'd', archive, ENV_INTERPOSERS], { stdio: 'inherit' });
|
|
125
|
+
}
|
|
126
|
+
// spidermonkey returns the folder with a target's pinned SpiderMonkey headers (include) and static
|
|
127
|
+
// libraries (lib), the ones upstream builds its own releases with.
|
|
128
|
+
function spidermonkey(key) {
|
|
129
|
+
const pin = toolchains.spidermonkey;
|
|
130
|
+
const archive = pin.targets[key];
|
|
131
|
+
if (!archive) throw new Error(`No pinned SpiderMonkey for ${key}`);
|
|
132
|
+
const linux = platforms[key][0] === 'linux';
|
|
133
|
+
return fetchPinned('SpiderMonkey', `https://github.com/${pin.repository}/releases/download/${pin.release}/${archive.asset}`,
|
|
134
|
+
archive, archive.asset.replace(/\.tar\.gz$/, linux ? '-without-env-interposers' : ''), null, linux ? dropEnvInterposers : null);
|
|
135
|
+
}
|
|
136
|
+
function xcode() {
|
|
137
|
+
try {
|
|
138
|
+
const [version, build] = execFileSync('xcodebuild', ['-version'], { encoding: 'utf8' }).trim().split('\n');
|
|
139
|
+
return `${version.replace(/^Xcode /, '')} (${build.replace(/^Build version /, '')})`;
|
|
140
|
+
} catch {
|
|
141
|
+
return null; // Command-line tools alone have no xcodebuild.
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
// cToolchain returns the C and C++ settings for a target and what they were made with. Only the
|
|
145
|
+
// targets being built and pinned must use the pinned toolchain; the host's is also used for upstream
|
|
146
|
+
// tests. SpiderMonkey's headers and libraries are found through CGO_CXXFLAGS and CGO_LDFLAGS, as
|
|
147
|
+
// upstream's own build finds them (.github/actions/spidermonkey); its mozjs package names the libraries.
|
|
148
|
+
function cToolchain(key, { pin = false } = {}) {
|
|
149
|
+
const check = pin ? pinned : () => {};
|
|
150
|
+
const [goos, goarch] = platforms[key];
|
|
151
|
+
const mozjs = spidermonkey(key);
|
|
152
|
+
// Quoted whole, as cgo splits these flags, so a folder with spaces stays one argument.
|
|
153
|
+
const include = `"-I${path.join(mozjs, 'include')}"`;
|
|
154
|
+
const lib = `"-L${path.join(mozjs, 'lib')}"`;
|
|
155
|
+
if (goos === 'darwin') {
|
|
156
|
+
if (process.platform !== 'darwin') throw new Error('Build macOS archives on macOS with Xcode');
|
|
157
|
+
const expected = `${toolchains.xcode.version} (${toolchains.xcode.build})`;
|
|
158
|
+
const actual = xcode();
|
|
159
|
+
check('Xcode', actual, expected,
|
|
160
|
+
`Select it with DEVELOPER_DIR=/Applications/Xcode_${toolchains.xcode.version}.app/Contents/Developer, or let CI make macOS pins (docs/studio-release.md).`);
|
|
161
|
+
const compiler = { xcode: actual, sdk: execFileSync('xcrun', ['--show-sdk-version'], { encoding: 'utf8' }).trim(),
|
|
162
|
+
clang: execFileSync('clang', ['--version'], { encoding: 'utf8' }).split('\n')[0] };
|
|
163
|
+
const flags = '-O2 -g -mmacosx-version-min=13.0';
|
|
164
|
+
return { compiler, env: { CC: 'clang', CXX: 'clang++', MACOSX_DEPLOYMENT_TARGET: '13.0',
|
|
165
|
+
CGO_CFLAGS: flags, CGO_CXXFLAGS: `${flags} ${include}`, CGO_LDFLAGS: `${flags} ${lib}` } };
|
|
166
|
+
}
|
|
167
|
+
if (goos === 'windows') {
|
|
168
|
+
const { cc, cxx, libs } = llvmMingw();
|
|
169
|
+
const clang = execFileSync(cc, ['--version'], { encoding: 'utf8' }).split('\n')[0];
|
|
170
|
+
const target = execFileSync(cc, ['-dumpmachine'], { encoding: 'utf8' }).trim();
|
|
171
|
+
if (target !== 'x86_64-w64-windows-gnu') throw new Error(`Windows needs llvm-mingw's x86_64 clang, ${cc} targets ${target}`);
|
|
172
|
+
check('Windows C compiler', clang, toolchains.llvmMingw.clang,
|
|
173
|
+
`Unset STUDIO_MINGW_CC to use the pinned llvm-mingw ${toolchains.llvmMingw.version}.`);
|
|
174
|
+
return { compiler: { llvmMingw: toolchains.llvmMingw.version, clang },
|
|
175
|
+
env: { CC: `"${cc}"`, CXX: `"${cxx}"`, CGO_CFLAGS: '-O2 -g', CGO_CXXFLAGS: `-O2 -g ${include}`,
|
|
176
|
+
CGO_LDFLAGS: `-O2 -g "-L${libs}" ${lib}` } };
|
|
177
|
+
}
|
|
178
|
+
const version = execFileSync(zig, ['version'], { encoding: 'utf8' }).trim();
|
|
179
|
+
check('Zig', version, toolchains.zig, 'Install it, or point STUDIO_ZIG at it.');
|
|
180
|
+
const triple = `${goarch === 'amd64' ? 'x86_64' : 'aarch64'}-linux-musl`;
|
|
181
|
+
return { compiler: { zig: version, target: triple },
|
|
182
|
+
env: { CC: `"${zig}" cc -target ${triple}`, CXX: `"${zig}" c++ -target ${triple}`,
|
|
183
|
+
CGO_CFLAGS: '-O2 -g', CGO_CXXFLAGS: `-O2 -g ${include}`, CGO_LDFLAGS: `-O2 -g ${lib}` } };
|
|
184
|
+
}
|
|
36
185
|
const go = (argv, extra = {}) => execFileSync('go', argv, { cwd: source, env: { ...env, ...extra }, stdio: 'inherit' });
|
|
37
186
|
const npm = argv => execFileSync(process.platform === 'win32' ? 'npm.cmd' : 'npm', argv,
|
|
38
187
|
{ cwd: path.join(source, 'web'), stdio: 'inherit', shell: process.platform === 'win32' });
|
|
39
188
|
const MANAGED_BY = 'magpie-gemini-web';
|
|
40
|
-
|
|
41
|
-
|
|
189
|
+
// An empty build ID: Go's default one hashes the build's inputs, the C compiler's path among them,
|
|
190
|
+
// so two builders' otherwise identical executables would differ in it.
|
|
191
|
+
const managedFlags = `-s -w -buildid= -X github.com/Mag1cFall/AIStudio2API/internal/app.version=v${release.version} -X github.com/Mag1cFall/AIStudio2API/internal/app.managedBy=${MANAGED_BY}`;
|
|
192
|
+
function linkFlags(goos) {
|
|
193
|
+
return goos === 'linux' ? "-linkmode external -extldflags '-static -Wl,--strip-all'"
|
|
194
|
+
: goos === 'windows' ? "-linkmode external -extldflags '-static -Wl,--no-insert-timestamp'" : '';
|
|
195
|
+
}
|
|
196
|
+
const buildRecords = {};
|
|
197
|
+
const compilers = {};
|
|
42
198
|
|
|
43
199
|
try {
|
|
200
|
+
// Check every target's toolchain before the slow part, so a wrong one fails in seconds.
|
|
201
|
+
for (const key of targets) cToolchain(key, { pin: true });
|
|
202
|
+
const hostToolchain = cToolchain(`${process.platform}-${process.arch}`).env;
|
|
44
203
|
let upstream = args.find(arg => arg.startsWith('--source='))?.slice(9);
|
|
45
204
|
if (upstream) {
|
|
46
205
|
upstream = path.resolve(upstream);
|
|
@@ -52,6 +211,12 @@ try {
|
|
|
52
211
|
}
|
|
53
212
|
execFileSync('git', ['-C', upstream, 'archive', '--format=tar', `--output=${path.join(scratch, 'upstream.tar')}`, release.upstream.commit]);
|
|
54
213
|
execFileSync('tar', ['-xf', 'upstream.tar', '-C', 'source'], { cwd: scratch });
|
|
214
|
+
const patches = (release.patches || []).map(name => {
|
|
215
|
+
const file = path.join(root, 'patches', name);
|
|
216
|
+
execFileSync('git', ['apply', '--check', file], { cwd: source, stdio: 'inherit' });
|
|
217
|
+
execFileSync('git', ['apply', file], { cwd: source, stdio: 'inherit' });
|
|
218
|
+
return { name, sha256: hash(file) };
|
|
219
|
+
});
|
|
55
220
|
// The plugin tells whether a sign-in will download Camoufox by the version this upstream takes.
|
|
56
221
|
const camoufox = fs.readFileSync(path.join(source, 'internal/camoufoxnative/download.go'), 'utf8')
|
|
57
222
|
.match(/const camoufoxRelease = "([^"]+)"/)?.[1];
|
|
@@ -64,30 +229,40 @@ try {
|
|
|
64
229
|
npm(['run', 'build']);
|
|
65
230
|
// Node's gzip differs between builders; Go's is the same everywhere (see studio-regzip.go).
|
|
66
231
|
go(['run', path.join(root, 'scripts/studio-regzip.go'), path.join(source, 'internal/webui/dist')]);
|
|
67
|
-
go(['test', '-mod=readonly', './cmd/...', './internal/...', '-count=1']);
|
|
232
|
+
go(['test', '-mod=readonly', `-ldflags=${linkFlags(process.platform === 'win32' ? 'windows' : process.platform)}`, './cmd/...', './internal/...', '-count=1'], hostToolchain);
|
|
68
233
|
const binaries = {};
|
|
69
|
-
|
|
234
|
+
const mismatches = [];
|
|
235
|
+
for (const key of targets) {
|
|
70
236
|
const [goos, goarch] = platforms[key];
|
|
71
237
|
const binary = path.join(scratch, goos === 'windows' ? 'aistudio2api.exe' : 'aistudio2api');
|
|
72
238
|
const asset = `aistudio2api-v${release.version}-${goos}-${goarch}.${goos === 'windows' ? 'zip' : 'tar.gz'}`;
|
|
73
239
|
console.log(`Building ${key}: ${asset}`);
|
|
74
|
-
|
|
240
|
+
const buildFlags = ['-mod=readonly', '-trimpath', '-buildvcs=false',
|
|
241
|
+
`-ldflags=${managedFlags} ${linkFlags(goos)}`, ...(goos === 'linux' ? ['-tags=netgo,osusergo'] : [])];
|
|
242
|
+
const toolchain = cToolchain(key);
|
|
243
|
+
buildRecords[key] = { flags: buildFlags, ...toolchain.env };
|
|
244
|
+
compilers[key] = toolchain.compiler;
|
|
245
|
+
go(['build', ...buildFlags, '-o', binary, './cmd/aistudio2api'], { ...toolchain.env, GOOS: goos, GOARCH: goarch });
|
|
75
246
|
go(['run', path.join(root, 'scripts/studio-archive.go'), path.join(output, asset), binary, path.join(source, 'LICENSE')]);
|
|
76
247
|
const sha256 = hash(path.join(output, asset));
|
|
77
248
|
const binarySha256 = hash(binary);
|
|
78
|
-
|
|
79
|
-
|
|
249
|
+
const pin = release.binaries[key];
|
|
250
|
+
if (verify && (pin?.sha256 !== sha256 || pin?.binarySha256 !== binarySha256 || pin?.asset !== asset)) {
|
|
251
|
+
mismatches.push(`${key}: built ${asset} ${sha256} (executable ${binarySha256}), pinned ${pin?.asset} ${pin?.sha256} (executable ${pin?.binarySha256})`);
|
|
80
252
|
}
|
|
81
253
|
binaries[key] = { asset, sha256, binarySha256 };
|
|
82
254
|
}
|
|
255
|
+
// Written even when verification fails: the record says what made the archives that differ.
|
|
256
|
+
const single = targets.length === 1 ? `-${targets[0]}` : '';
|
|
83
257
|
fs.copyFileSync(path.join(source, 'LICENSE'), path.join(output, 'LICENSE-AISTUDIO'));
|
|
84
|
-
fs.writeFileSync(path.join(output,
|
|
85
|
-
version: release.version, upstream: release.upstream, goVersion: release.goVersion,
|
|
86
|
-
nodeVersion: process.version, managedBy: MANAGED_BY,
|
|
258
|
+
fs.writeFileSync(path.join(output, `build${single}.json`), JSON.stringify({
|
|
259
|
+
version: release.version, upstream: release.upstream, goVersion: release.goVersion, toolchains,
|
|
260
|
+
nodeVersion: process.version, managedBy: MANAGED_BY, builds: buildRecords, compiler: compilers, patches, binaries
|
|
87
261
|
}, null, 2) + '\n');
|
|
88
|
-
fs.writeFileSync(path.join(output,
|
|
262
|
+
fs.writeFileSync(path.join(output, `SHA256SUMS${single}.txt`),
|
|
89
263
|
Object.values(binaries).map(pin => `${pin.sha256} ${pin.asset}\n`).join(''));
|
|
90
|
-
if (
|
|
264
|
+
if (mismatches.length) throw new Error(`Release pin mismatch:\n ${mismatches.join('\n ')}`);
|
|
265
|
+
if (writePins) fs.writeFileSync(pinPath, JSON.stringify({ ...release, binaries: { ...release.binaries, ...binaries } }, null, 2) + '\n');
|
|
91
266
|
console.log(`Managed Studio archives ready: ${output}`);
|
|
92
267
|
} finally {
|
|
93
268
|
fs.rmSync(scratch, { recursive: true, force: true });
|