@hasna/brains 0.0.6 → 0.0.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +190 -0
- package/dist/cli/index.js +1172 -809
- package/dist/db/index.d.ts.map +1 -1
- package/dist/index.js +4294 -142
- package/dist/lib/config.d.ts +11 -0
- package/dist/lib/config.d.ts.map +1 -0
- package/dist/lib/gatherers/index.d.ts +1 -0
- package/dist/lib/gatherers/index.d.ts.map +1 -1
- package/dist/lib/index.d.ts +3 -0
- package/dist/lib/index.d.ts.map +1 -1
- package/dist/lib/providers/openai.d.ts +2 -0
- package/dist/lib/providers/openai.d.ts.map +1 -1
- package/dist/lib/providers/thinker-labs.d.ts +1 -0
- package/dist/lib/providers/thinker-labs.d.ts.map +1 -1
- package/dist/lib/retry.d.ts +7 -0
- package/dist/lib/retry.d.ts.map +1 -0
- package/dist/lib/schemas.d.ts +87 -0
- package/dist/lib/schemas.d.ts.map +1 -0
- package/dist/mcp/index.d.ts.map +1 -1
- package/dist/mcp/index.js +4585 -196
- package/dist/server/index.d.ts +1 -1
- package/dist/server/index.d.ts.map +1 -1
- package/dist/server/index.js +4309 -9
- package/drizzle/0000_foamy_daimon_hellstrom.sql +36 -0
- package/drizzle/meta/0000_snapshot.json +259 -0
- package/drizzle/meta/_journal.json +13 -0
- package/package.json +6 -2
package/dist/cli/index.js
CHANGED
|
@@ -5,45 +5,28 @@ var __getProtoOf = Object.getPrototypeOf;
|
|
|
5
5
|
var __defProp = Object.defineProperty;
|
|
6
6
|
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
7
7
|
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
8
|
-
function __accessProp(key) {
|
|
9
|
-
return this[key];
|
|
10
|
-
}
|
|
11
|
-
var __toESMCache_node;
|
|
12
|
-
var __toESMCache_esm;
|
|
13
8
|
var __toESM = (mod, isNodeMode, target) => {
|
|
14
|
-
var canCache = mod != null && typeof mod === "object";
|
|
15
|
-
if (canCache) {
|
|
16
|
-
var cache = isNodeMode ? __toESMCache_node ??= new WeakMap : __toESMCache_esm ??= new WeakMap;
|
|
17
|
-
var cached = cache.get(mod);
|
|
18
|
-
if (cached)
|
|
19
|
-
return cached;
|
|
20
|
-
}
|
|
21
9
|
target = mod != null ? __create(__getProtoOf(mod)) : {};
|
|
22
10
|
const to = isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target;
|
|
23
11
|
for (let key of __getOwnPropNames(mod))
|
|
24
12
|
if (!__hasOwnProp.call(to, key))
|
|
25
13
|
__defProp(to, key, {
|
|
26
|
-
get:
|
|
14
|
+
get: () => mod[key],
|
|
27
15
|
enumerable: true
|
|
28
16
|
});
|
|
29
|
-
if (canCache)
|
|
30
|
-
cache.set(mod, to);
|
|
31
17
|
return to;
|
|
32
18
|
};
|
|
33
19
|
var __commonJS = (cb, mod) => () => (mod || cb((mod = { exports: {} }).exports, mod), mod.exports);
|
|
34
|
-
var __returnValue = (v) => v;
|
|
35
|
-
function __exportSetter(name, newValue) {
|
|
36
|
-
this[name] = __returnValue.bind(null, newValue);
|
|
37
|
-
}
|
|
38
20
|
var __export = (target, all) => {
|
|
39
21
|
for (var name in all)
|
|
40
22
|
__defProp(target, name, {
|
|
41
23
|
get: all[name],
|
|
42
24
|
enumerable: true,
|
|
43
25
|
configurable: true,
|
|
44
|
-
set:
|
|
26
|
+
set: (newValue) => all[name] = () => newValue
|
|
45
27
|
});
|
|
46
28
|
};
|
|
29
|
+
var __esm = (fn, res) => () => (fn && (res = fn(fn = 0)), res);
|
|
47
30
|
var __require = import.meta.require;
|
|
48
31
|
|
|
49
32
|
// node_modules/commander/lib/error.js
|
|
@@ -2080,167 +2063,513 @@ var require_commander = __commonJS((exports) => {
|
|
|
2080
2063
|
exports.InvalidOptionArgumentError = InvalidArgumentError;
|
|
2081
2064
|
});
|
|
2082
2065
|
|
|
2083
|
-
//
|
|
2084
|
-
var
|
|
2085
|
-
|
|
2086
|
-
|
|
2087
|
-
|
|
2088
|
-
|
|
2089
|
-
|
|
2090
|
-
|
|
2091
|
-
|
|
2092
|
-
|
|
2093
|
-
Command,
|
|
2094
|
-
Argument,
|
|
2095
|
-
Option,
|
|
2096
|
-
Help
|
|
2097
|
-
} = import__.default;
|
|
2066
|
+
// src/lib/gatherers/todos.ts
|
|
2067
|
+
var exports_todos = {};
|
|
2068
|
+
__export(exports_todos, {
|
|
2069
|
+
gatherFromTodos: () => gatherFromTodos
|
|
2070
|
+
});
|
|
2071
|
+
import { Database as Database3 } from "bun:sqlite";
|
|
2072
|
+
import { homedir as homedir3 } from "os";
|
|
2073
|
+
import { join as join3 } from "path";
|
|
2074
|
+
function taskToCreateExample(task) {
|
|
2075
|
+
const userMsg = `Create a task: ${task.title}${task.description ? `
|
|
2098
2076
|
|
|
2099
|
-
|
|
2100
|
-
|
|
2101
|
-
|
|
2102
|
-
|
|
2103
|
-
|
|
2104
|
-
|
|
2105
|
-
|
|
2106
|
-
|
|
2107
|
-
|
|
2108
|
-
}
|
|
2109
|
-
|
|
2110
|
-
|
|
2111
|
-
|
|
2112
|
-
|
|
2113
|
-
|
|
2114
|
-
|
|
2115
|
-
|
|
2116
|
-
return true;
|
|
2117
|
-
}
|
|
2118
|
-
cls = Object.getPrototypeOf(cls);
|
|
2119
|
-
}
|
|
2120
|
-
}
|
|
2121
|
-
return false;
|
|
2077
|
+
Description: ${task.description}` : ""}`;
|
|
2078
|
+
const taskDetails = {
|
|
2079
|
+
id: task.short_id ?? task.id,
|
|
2080
|
+
title: task.title,
|
|
2081
|
+
description: task.description ?? "",
|
|
2082
|
+
status: task.status,
|
|
2083
|
+
priority: task.priority,
|
|
2084
|
+
tags: JSON.parse(task.tags ?? "[]"),
|
|
2085
|
+
created_at: task.created_at
|
|
2086
|
+
};
|
|
2087
|
+
return {
|
|
2088
|
+
messages: [
|
|
2089
|
+
{ role: "system", content: SYSTEM_PROMPT },
|
|
2090
|
+
{ role: "user", content: userMsg },
|
|
2091
|
+
{ role: "assistant", content: `Created task: ${JSON.stringify(taskDetails, null, 2)}` }
|
|
2092
|
+
]
|
|
2093
|
+
};
|
|
2122
2094
|
}
|
|
2123
|
-
|
|
2124
|
-
|
|
2125
|
-
|
|
2126
|
-
|
|
2127
|
-
|
|
2128
|
-
|
|
2129
|
-
|
|
2130
|
-
|
|
2131
|
-
|
|
2132
|
-
|
|
2133
|
-
|
|
2134
|
-
this.onUpdateFn = config.onUpdateFn;
|
|
2135
|
-
this.hasDefault = config.hasDefault;
|
|
2136
|
-
this.primary = config.primaryKey;
|
|
2137
|
-
this.isUnique = config.isUnique;
|
|
2138
|
-
this.uniqueName = config.uniqueName;
|
|
2139
|
-
this.uniqueType = config.uniqueType;
|
|
2140
|
-
this.dataType = config.dataType;
|
|
2141
|
-
this.columnType = config.columnType;
|
|
2142
|
-
this.generated = config.generated;
|
|
2143
|
-
this.generatedIdentity = config.generatedIdentity;
|
|
2144
|
-
}
|
|
2145
|
-
static [entityKind] = "Column";
|
|
2146
|
-
name;
|
|
2147
|
-
keyAsName;
|
|
2148
|
-
primary;
|
|
2149
|
-
notNull;
|
|
2150
|
-
default;
|
|
2151
|
-
defaultFn;
|
|
2152
|
-
onUpdateFn;
|
|
2153
|
-
hasDefault;
|
|
2154
|
-
isUnique;
|
|
2155
|
-
uniqueName;
|
|
2156
|
-
uniqueType;
|
|
2157
|
-
dataType;
|
|
2158
|
-
columnType;
|
|
2159
|
-
enumValues = undefined;
|
|
2160
|
-
generated = undefined;
|
|
2161
|
-
generatedIdentity = undefined;
|
|
2162
|
-
config;
|
|
2163
|
-
mapFromDriverValue(value) {
|
|
2164
|
-
return value;
|
|
2165
|
-
}
|
|
2166
|
-
mapToDriverValue(value) {
|
|
2167
|
-
return value;
|
|
2168
|
-
}
|
|
2169
|
-
shouldDisableInsert() {
|
|
2170
|
-
return this.config.generated !== undefined && this.config.generated.type !== "byDefault";
|
|
2171
|
-
}
|
|
2095
|
+
function taskToStatusUpdateExample(task) {
|
|
2096
|
+
if (!task.completed_at && task.status === "pending")
|
|
2097
|
+
return null;
|
|
2098
|
+
const id = task.short_id ?? task.id;
|
|
2099
|
+
return {
|
|
2100
|
+
messages: [
|
|
2101
|
+
{ role: "system", content: SYSTEM_PROMPT },
|
|
2102
|
+
{ role: "user", content: `Mark task ${id} as ${task.status}` },
|
|
2103
|
+
{ role: "assistant", content: `Task ${id} has been updated to status: ${task.status}. ${task.completed_at ? `Completed at: ${task.completed_at}` : ""}`.trim() }
|
|
2104
|
+
]
|
|
2105
|
+
};
|
|
2172
2106
|
}
|
|
2173
|
-
|
|
2174
|
-
|
|
2175
|
-
|
|
2176
|
-
|
|
2177
|
-
|
|
2178
|
-
|
|
2179
|
-
|
|
2180
|
-
|
|
2181
|
-
|
|
2182
|
-
|
|
2183
|
-
|
|
2184
|
-
|
|
2185
|
-
|
|
2186
|
-
|
|
2187
|
-
|
|
2188
|
-
|
|
2189
|
-
|
|
2190
|
-
|
|
2191
|
-
|
|
2107
|
+
function taskToSearchExample(tasks, query) {
|
|
2108
|
+
const matched = tasks.filter((t) => t.title.toLowerCase().includes(query.toLowerCase())).slice(0, 5);
|
|
2109
|
+
return {
|
|
2110
|
+
messages: [
|
|
2111
|
+
{ role: "system", content: SYSTEM_PROMPT },
|
|
2112
|
+
{ role: "user", content: `Search tasks for: "${query}"` },
|
|
2113
|
+
{
|
|
2114
|
+
role: "assistant",
|
|
2115
|
+
content: matched.length > 0 ? `Found ${matched.length} task(s):
|
|
2116
|
+
${matched.map((t) => `- [${t.short_id ?? t.id}] ${t.title} (${t.status})`).join(`
|
|
2117
|
+
`)}` : `No tasks found matching "${query}".`
|
|
2118
|
+
}
|
|
2119
|
+
]
|
|
2120
|
+
};
|
|
2121
|
+
}
|
|
2122
|
+
async function gatherFromTodos(options = {}) {
|
|
2123
|
+
const dbPath = join3(homedir3(), ".todos", "todos.db");
|
|
2124
|
+
const db = new Database3(dbPath, { readonly: true, create: false });
|
|
2125
|
+
try {
|
|
2126
|
+
let query = "SELECT * FROM tasks WHERE 1=1";
|
|
2127
|
+
const params = [];
|
|
2128
|
+
if (options.since) {
|
|
2129
|
+
query += " AND created_at >= ?";
|
|
2130
|
+
params.push(options.since.toISOString());
|
|
2131
|
+
}
|
|
2132
|
+
query += " ORDER BY created_at DESC";
|
|
2133
|
+
if (options.limit) {
|
|
2134
|
+
query += " LIMIT ?";
|
|
2135
|
+
params.push(options.limit * 2);
|
|
2136
|
+
}
|
|
2137
|
+
const tasks = db.query(query).all(...params);
|
|
2138
|
+
const examples = [];
|
|
2139
|
+
for (const task of tasks) {
|
|
2140
|
+
examples.push(taskToCreateExample(task));
|
|
2141
|
+
const statusEx = taskToStatusUpdateExample(task);
|
|
2142
|
+
if (statusEx)
|
|
2143
|
+
examples.push(statusEx);
|
|
2144
|
+
}
|
|
2145
|
+
const searchTerms = ["urgent", "fix", "implement", "create", "update", "review"];
|
|
2146
|
+
for (const term of searchTerms) {
|
|
2147
|
+
examples.push(taskToSearchExample(tasks, term));
|
|
2148
|
+
}
|
|
2149
|
+
const finalExamples = options.limit ? examples.slice(0, options.limit) : examples;
|
|
2150
|
+
return {
|
|
2151
|
+
source: "todos",
|
|
2152
|
+
examples: finalExamples,
|
|
2153
|
+
count: finalExamples.length
|
|
2192
2154
|
};
|
|
2193
|
-
}
|
|
2194
|
-
|
|
2195
|
-
return this;
|
|
2196
|
-
}
|
|
2197
|
-
notNull() {
|
|
2198
|
-
this.config.notNull = true;
|
|
2199
|
-
return this;
|
|
2200
|
-
}
|
|
2201
|
-
default(value) {
|
|
2202
|
-
this.config.default = value;
|
|
2203
|
-
this.config.hasDefault = true;
|
|
2204
|
-
return this;
|
|
2205
|
-
}
|
|
2206
|
-
$defaultFn(fn) {
|
|
2207
|
-
this.config.defaultFn = fn;
|
|
2208
|
-
this.config.hasDefault = true;
|
|
2209
|
-
return this;
|
|
2210
|
-
}
|
|
2211
|
-
$default = this.$defaultFn;
|
|
2212
|
-
$onUpdateFn(fn) {
|
|
2213
|
-
this.config.onUpdateFn = fn;
|
|
2214
|
-
this.config.hasDefault = true;
|
|
2215
|
-
return this;
|
|
2216
|
-
}
|
|
2217
|
-
$onUpdate = this.$onUpdateFn;
|
|
2218
|
-
primaryKey() {
|
|
2219
|
-
this.config.primaryKey = true;
|
|
2220
|
-
this.config.notNull = true;
|
|
2221
|
-
return this;
|
|
2222
|
-
}
|
|
2223
|
-
setName(name) {
|
|
2224
|
-
if (this.config.name !== "")
|
|
2225
|
-
return;
|
|
2226
|
-
this.config.name = name;
|
|
2155
|
+
} finally {
|
|
2156
|
+
db.close();
|
|
2227
2157
|
}
|
|
2228
2158
|
}
|
|
2159
|
+
var SYSTEM_PROMPT = "You are a task management assistant that helps users create, update, search, and manage tasks and projects.";
|
|
2160
|
+
var init_todos = () => {};
|
|
2229
2161
|
|
|
2230
|
-
//
|
|
2231
|
-
var
|
|
2232
|
-
|
|
2233
|
-
|
|
2234
|
-
|
|
2235
|
-
|
|
2236
|
-
}
|
|
2162
|
+
// src/lib/gatherers/mementos.ts
|
|
2163
|
+
var exports_mementos = {};
|
|
2164
|
+
__export(exports_mementos, {
|
|
2165
|
+
gatherFromMementos: () => gatherFromMementos
|
|
2166
|
+
});
|
|
2167
|
+
import { Database as Database4 } from "bun:sqlite";
|
|
2168
|
+
import { homedir as homedir4 } from "os";
|
|
2169
|
+
import { join as join4 } from "path";
|
|
2170
|
+
function memoryToRecallExample(memory) {
|
|
2171
|
+
return {
|
|
2172
|
+
messages: [
|
|
2173
|
+
{ role: "system", content: SYSTEM_PROMPT2 },
|
|
2174
|
+
{ role: "user", content: `What do you remember about "${memory.key}"?` },
|
|
2175
|
+
{
|
|
2176
|
+
role: "assistant",
|
|
2177
|
+
content: memory.summary ? `${memory.value}
|
|
2237
2178
|
|
|
2238
|
-
|
|
2239
|
-
|
|
2240
|
-
|
|
2179
|
+
Summary: ${memory.summary}` : memory.value
|
|
2180
|
+
}
|
|
2181
|
+
]
|
|
2182
|
+
};
|
|
2241
2183
|
}
|
|
2242
|
-
|
|
2243
|
-
|
|
2184
|
+
function memoryToSaveExample(memory) {
|
|
2185
|
+
const tags = JSON.parse(memory.tags ?? "[]");
|
|
2186
|
+
return {
|
|
2187
|
+
messages: [
|
|
2188
|
+
{ role: "system", content: SYSTEM_PROMPT2 },
|
|
2189
|
+
{
|
|
2190
|
+
role: "user",
|
|
2191
|
+
content: `Remember this for me: ${memory.key} = ${memory.value}${tags.length ? ` (tags: ${tags.join(", ")})` : ""}`
|
|
2192
|
+
},
|
|
2193
|
+
{
|
|
2194
|
+
role: "assistant",
|
|
2195
|
+
content: `Saved to memory: "${memory.key}" with ${memory.category} category, importance ${memory.importance}/10, scope: ${memory.scope}.`
|
|
2196
|
+
}
|
|
2197
|
+
]
|
|
2198
|
+
};
|
|
2199
|
+
}
|
|
2200
|
+
function memoryToSearchExample(memories, category) {
|
|
2201
|
+
const matched = memories.filter((m) => m.category === category && m.status === "active").slice(0, 5);
|
|
2202
|
+
return {
|
|
2203
|
+
messages: [
|
|
2204
|
+
{ role: "system", content: SYSTEM_PROMPT2 },
|
|
2205
|
+
{ role: "user", content: `What ${category} memories do you have?` },
|
|
2206
|
+
{
|
|
2207
|
+
role: "assistant",
|
|
2208
|
+
content: matched.length > 0 ? `Here are my ${category} memories:
|
|
2209
|
+
${matched.map((m) => `- ${m.key}: ${m.value.slice(0, 120)}${m.value.length > 120 ? "..." : ""}`).join(`
|
|
2210
|
+
`)}` : `I don't have any ${category} memories stored yet.`
|
|
2211
|
+
}
|
|
2212
|
+
]
|
|
2213
|
+
};
|
|
2214
|
+
}
|
|
2215
|
+
async function gatherFromMementos(options = {}) {
|
|
2216
|
+
const dbPath = join4(homedir4(), ".mementos", "mementos.db");
|
|
2217
|
+
const db = new Database4(dbPath, { readonly: true, create: false });
|
|
2218
|
+
try {
|
|
2219
|
+
let query = "SELECT * FROM memories WHERE status = 'active'";
|
|
2220
|
+
const params = [];
|
|
2221
|
+
if (options.since) {
|
|
2222
|
+
query += " AND created_at >= ?";
|
|
2223
|
+
params.push(options.since.toISOString());
|
|
2224
|
+
}
|
|
2225
|
+
query += " ORDER BY importance DESC, created_at DESC";
|
|
2226
|
+
if (options.limit) {
|
|
2227
|
+
query += " LIMIT ?";
|
|
2228
|
+
params.push(options.limit * 3);
|
|
2229
|
+
}
|
|
2230
|
+
const memories = db.query(query).all(...params);
|
|
2231
|
+
const examples = [];
|
|
2232
|
+
for (const memory of memories) {
|
|
2233
|
+
examples.push(memoryToRecallExample(memory));
|
|
2234
|
+
examples.push(memoryToSaveExample(memory));
|
|
2235
|
+
}
|
|
2236
|
+
const categories = [...new Set(memories.map((m) => m.category))];
|
|
2237
|
+
for (const category of categories) {
|
|
2238
|
+
examples.push(memoryToSearchExample(memories, category));
|
|
2239
|
+
}
|
|
2240
|
+
const finalExamples = options.limit ? examples.slice(0, options.limit) : examples;
|
|
2241
|
+
return {
|
|
2242
|
+
source: "mementos",
|
|
2243
|
+
examples: finalExamples,
|
|
2244
|
+
count: finalExamples.length
|
|
2245
|
+
};
|
|
2246
|
+
} finally {
|
|
2247
|
+
db.close();
|
|
2248
|
+
}
|
|
2249
|
+
}
|
|
2250
|
+
var SYSTEM_PROMPT2 = "You are an AI assistant with persistent memory. You can remember and recall information across sessions to provide better, more personalized assistance.";
|
|
2251
|
+
var init_mementos = () => {};
|
|
2252
|
+
|
|
2253
|
+
// src/lib/gatherers/conversations.ts
|
|
2254
|
+
var exports_conversations = {};
|
|
2255
|
+
__export(exports_conversations, {
|
|
2256
|
+
gatherFromConversations: () => gatherFromConversations
|
|
2257
|
+
});
|
|
2258
|
+
import { Database as Database5 } from "bun:sqlite";
|
|
2259
|
+
import { homedir as homedir5 } from "os";
|
|
2260
|
+
import { join as join5 } from "path";
|
|
2261
|
+
function windowToExample(window2) {
|
|
2262
|
+
if (window2.length < 2)
|
|
2263
|
+
return null;
|
|
2264
|
+
const messages = [
|
|
2265
|
+
{ role: "system", content: SYSTEM_PROMPT3 }
|
|
2266
|
+
];
|
|
2267
|
+
for (let i = 0;i < window2.length - 1; i++) {
|
|
2268
|
+
const msg = window2[i];
|
|
2269
|
+
if (!msg)
|
|
2270
|
+
continue;
|
|
2271
|
+
const role = i % 2 === 0 ? "user" : "assistant";
|
|
2272
|
+
messages.push({
|
|
2273
|
+
role,
|
|
2274
|
+
content: `[${msg.from_agent} \u2192 ${msg.to_agent ?? msg.space ?? "all"}]: ${msg.content}`
|
|
2275
|
+
});
|
|
2276
|
+
}
|
|
2277
|
+
const last = window2[window2.length - 1];
|
|
2278
|
+
if (!last)
|
|
2279
|
+
return null;
|
|
2280
|
+
messages.push({
|
|
2281
|
+
role: "assistant",
|
|
2282
|
+
content: `[${last.from_agent} \u2192 ${last.to_agent ?? last.space ?? "all"}]: ${last.content}`
|
|
2283
|
+
});
|
|
2284
|
+
return { messages };
|
|
2285
|
+
}
|
|
2286
|
+
async function gatherFromConversations(options = {}) {
|
|
2287
|
+
const dbPath = join5(homedir5(), ".conversations", "messages.db");
|
|
2288
|
+
const db = new Database5(dbPath, { readonly: true, create: false });
|
|
2289
|
+
try {
|
|
2290
|
+
let query = "SELECT * FROM messages WHERE 1=1";
|
|
2291
|
+
const params = [];
|
|
2292
|
+
if (options.since) {
|
|
2293
|
+
query += " AND created_at >= ?";
|
|
2294
|
+
params.push(options.since.toISOString());
|
|
2295
|
+
}
|
|
2296
|
+
query += " ORDER BY session_id, created_at ASC";
|
|
2297
|
+
const allMessages = db.query(query).all(...params);
|
|
2298
|
+
const sessions = new Map;
|
|
2299
|
+
for (const msg of allMessages) {
|
|
2300
|
+
const msgs = sessions.get(msg.session_id) ?? [];
|
|
2301
|
+
msgs.push(msg);
|
|
2302
|
+
sessions.set(msg.session_id, msgs);
|
|
2303
|
+
}
|
|
2304
|
+
const examples = [];
|
|
2305
|
+
const windowSize = 4;
|
|
2306
|
+
for (const [, sessionMsgs] of sessions) {
|
|
2307
|
+
if (sessionMsgs.length < 2)
|
|
2308
|
+
continue;
|
|
2309
|
+
for (let start = 0;start <= sessionMsgs.length - 2; start++) {
|
|
2310
|
+
const end = Math.min(start + windowSize, sessionMsgs.length);
|
|
2311
|
+
const window2 = sessionMsgs.slice(start, end);
|
|
2312
|
+
const example = windowToExample(window2);
|
|
2313
|
+
if (example)
|
|
2314
|
+
examples.push(example);
|
|
2315
|
+
}
|
|
2316
|
+
}
|
|
2317
|
+
const finalExamples = options.limit ? examples.slice(0, options.limit) : examples;
|
|
2318
|
+
return {
|
|
2319
|
+
source: "conversations",
|
|
2320
|
+
examples: finalExamples,
|
|
2321
|
+
count: finalExamples.length
|
|
2322
|
+
};
|
|
2323
|
+
} finally {
|
|
2324
|
+
db.close();
|
|
2325
|
+
}
|
|
2326
|
+
}
|
|
2327
|
+
var SYSTEM_PROMPT3 = "You are a helpful AI assistant participating in multi-agent conversations. You communicate clearly and collaboratively with other agents and users.";
|
|
2328
|
+
var init_conversations = () => {};
|
|
2329
|
+
|
|
2330
|
+
// src/lib/gatherers/sessions.ts
|
|
2331
|
+
var exports_sessions2 = {};
|
|
2332
|
+
__export(exports_sessions2, {
|
|
2333
|
+
gatherFromSessions: () => gatherFromSessions
|
|
2334
|
+
});
|
|
2335
|
+
import { readdir, readFile, stat } from "fs/promises";
|
|
2336
|
+
import { existsSync as existsSync2 } from "fs";
|
|
2337
|
+
import { join as join6 } from "path";
|
|
2338
|
+
import { homedir as homedir6 } from "os";
|
|
2339
|
+
function extractText(content) {
|
|
2340
|
+
if (typeof content === "string")
|
|
2341
|
+
return content;
|
|
2342
|
+
return content.filter((c) => c.type === "text" && c.text).map((c) => c.text).join(`
|
|
2343
|
+
`).trim();
|
|
2344
|
+
}
|
|
2345
|
+
async function gatherFromSessions(options = {}) {
|
|
2346
|
+
const { limit: limit2 = 1000 } = options;
|
|
2347
|
+
const examples = [];
|
|
2348
|
+
const claudeDir = join6(homedir6(), ".claude", "projects");
|
|
2349
|
+
if (!existsSync2(claudeDir)) {
|
|
2350
|
+
return { source: "sessions", examples: [], count: 0 };
|
|
2351
|
+
}
|
|
2352
|
+
const projectDirs = await readdir(claudeDir).catch(() => []);
|
|
2353
|
+
for (const projectDir of projectDirs) {
|
|
2354
|
+
if (examples.length >= limit2)
|
|
2355
|
+
break;
|
|
2356
|
+
const projectPath = join6(claudeDir, projectDir);
|
|
2357
|
+
const files = await readdir(projectPath).catch(() => []);
|
|
2358
|
+
for (const file of files) {
|
|
2359
|
+
if (examples.length >= limit2)
|
|
2360
|
+
break;
|
|
2361
|
+
if (!file.endsWith(".jsonl"))
|
|
2362
|
+
continue;
|
|
2363
|
+
const filePath = join6(projectPath, file);
|
|
2364
|
+
if (options.since) {
|
|
2365
|
+
const fileStat = await stat(filePath).catch(() => null);
|
|
2366
|
+
if (fileStat && fileStat.mtime < options.since)
|
|
2367
|
+
continue;
|
|
2368
|
+
}
|
|
2369
|
+
const content = await readFile(filePath, "utf-8").catch(() => "");
|
|
2370
|
+
if (!content.trim())
|
|
2371
|
+
continue;
|
|
2372
|
+
const lines = content.trim().split(`
|
|
2373
|
+
`);
|
|
2374
|
+
const turns = [];
|
|
2375
|
+
for (const line of lines) {
|
|
2376
|
+
try {
|
|
2377
|
+
const entry = JSON.parse(line);
|
|
2378
|
+
if ((entry.type === "user" || entry.type === "human") && entry.message?.content) {
|
|
2379
|
+
const text2 = extractText(entry.message.content);
|
|
2380
|
+
if (text2.trim())
|
|
2381
|
+
turns.push({ role: "user", content: text2.trim() });
|
|
2382
|
+
} else if (entry.type === "assistant" && entry.message?.content) {
|
|
2383
|
+
const text2 = extractText(entry.message.content);
|
|
2384
|
+
if (text2.trim())
|
|
2385
|
+
turns.push({ role: "assistant", content: text2.trim() });
|
|
2386
|
+
}
|
|
2387
|
+
} catch {}
|
|
2388
|
+
}
|
|
2389
|
+
const windowSize = 6;
|
|
2390
|
+
for (let start = 0;start < turns.length - 1 && examples.length < limit2; start++) {
|
|
2391
|
+
const window2 = turns.slice(start, start + windowSize);
|
|
2392
|
+
if (!window2[0] || window2[0].role !== "user")
|
|
2393
|
+
continue;
|
|
2394
|
+
const lastAssistantIdx = window2.map((t) => t.role).lastIndexOf("assistant");
|
|
2395
|
+
if (lastAssistantIdx < 1)
|
|
2396
|
+
continue;
|
|
2397
|
+
const usedTurns = window2.slice(0, lastAssistantIdx + 1);
|
|
2398
|
+
examples.push({
|
|
2399
|
+
messages: [
|
|
2400
|
+
{ role: "system", content: SYSTEM_PROMPT4 },
|
|
2401
|
+
...usedTurns
|
|
2402
|
+
]
|
|
2403
|
+
});
|
|
2404
|
+
}
|
|
2405
|
+
}
|
|
2406
|
+
}
|
|
2407
|
+
return { source: "sessions", examples, count: examples.length };
|
|
2408
|
+
}
|
|
2409
|
+
var SYSTEM_PROMPT4 = "You are Claude Code, an AI assistant built by Anthropic that helps developers with coding, architecture, debugging, and software engineering tasks.";
|
|
2410
|
+
var init_sessions = () => {};
|
|
2411
|
+
|
|
2412
|
+
// node_modules/commander/esm.mjs
|
|
2413
|
+
var import__ = __toESM(require_commander(), 1);
|
|
2414
|
+
var {
|
|
2415
|
+
program,
|
|
2416
|
+
createCommand,
|
|
2417
|
+
createArgument,
|
|
2418
|
+
createOption,
|
|
2419
|
+
CommanderError,
|
|
2420
|
+
InvalidArgumentError,
|
|
2421
|
+
InvalidOptionArgumentError,
|
|
2422
|
+
Command,
|
|
2423
|
+
Argument,
|
|
2424
|
+
Option,
|
|
2425
|
+
Help
|
|
2426
|
+
} = import__.default;
|
|
2427
|
+
|
|
2428
|
+
// node_modules/drizzle-orm/entity.js
|
|
2429
|
+
var entityKind = Symbol.for("drizzle:entityKind");
|
|
2430
|
+
var hasOwnEntityKind = Symbol.for("drizzle:hasOwnEntityKind");
|
|
2431
|
+
function is(value, type) {
|
|
2432
|
+
if (!value || typeof value !== "object") {
|
|
2433
|
+
return false;
|
|
2434
|
+
}
|
|
2435
|
+
if (value instanceof type) {
|
|
2436
|
+
return true;
|
|
2437
|
+
}
|
|
2438
|
+
if (!Object.prototype.hasOwnProperty.call(type, entityKind)) {
|
|
2439
|
+
throw new Error(`Class "${type.name ?? "<unknown>"}" doesn't look like a Drizzle entity. If this is incorrect and the class is provided by Drizzle, please report this as a bug.`);
|
|
2440
|
+
}
|
|
2441
|
+
let cls = Object.getPrototypeOf(value).constructor;
|
|
2442
|
+
if (cls) {
|
|
2443
|
+
while (cls) {
|
|
2444
|
+
if (entityKind in cls && cls[entityKind] === type[entityKind]) {
|
|
2445
|
+
return true;
|
|
2446
|
+
}
|
|
2447
|
+
cls = Object.getPrototypeOf(cls);
|
|
2448
|
+
}
|
|
2449
|
+
}
|
|
2450
|
+
return false;
|
|
2451
|
+
}
|
|
2452
|
+
|
|
2453
|
+
// node_modules/drizzle-orm/column.js
|
|
2454
|
+
class Column {
|
|
2455
|
+
constructor(table, config) {
|
|
2456
|
+
this.table = table;
|
|
2457
|
+
this.config = config;
|
|
2458
|
+
this.name = config.name;
|
|
2459
|
+
this.keyAsName = config.keyAsName;
|
|
2460
|
+
this.notNull = config.notNull;
|
|
2461
|
+
this.default = config.default;
|
|
2462
|
+
this.defaultFn = config.defaultFn;
|
|
2463
|
+
this.onUpdateFn = config.onUpdateFn;
|
|
2464
|
+
this.hasDefault = config.hasDefault;
|
|
2465
|
+
this.primary = config.primaryKey;
|
|
2466
|
+
this.isUnique = config.isUnique;
|
|
2467
|
+
this.uniqueName = config.uniqueName;
|
|
2468
|
+
this.uniqueType = config.uniqueType;
|
|
2469
|
+
this.dataType = config.dataType;
|
|
2470
|
+
this.columnType = config.columnType;
|
|
2471
|
+
this.generated = config.generated;
|
|
2472
|
+
this.generatedIdentity = config.generatedIdentity;
|
|
2473
|
+
}
|
|
2474
|
+
static [entityKind] = "Column";
|
|
2475
|
+
name;
|
|
2476
|
+
keyAsName;
|
|
2477
|
+
primary;
|
|
2478
|
+
notNull;
|
|
2479
|
+
default;
|
|
2480
|
+
defaultFn;
|
|
2481
|
+
onUpdateFn;
|
|
2482
|
+
hasDefault;
|
|
2483
|
+
isUnique;
|
|
2484
|
+
uniqueName;
|
|
2485
|
+
uniqueType;
|
|
2486
|
+
dataType;
|
|
2487
|
+
columnType;
|
|
2488
|
+
enumValues = undefined;
|
|
2489
|
+
generated = undefined;
|
|
2490
|
+
generatedIdentity = undefined;
|
|
2491
|
+
config;
|
|
2492
|
+
mapFromDriverValue(value) {
|
|
2493
|
+
return value;
|
|
2494
|
+
}
|
|
2495
|
+
mapToDriverValue(value) {
|
|
2496
|
+
return value;
|
|
2497
|
+
}
|
|
2498
|
+
shouldDisableInsert() {
|
|
2499
|
+
return this.config.generated !== undefined && this.config.generated.type !== "byDefault";
|
|
2500
|
+
}
|
|
2501
|
+
}
|
|
2502
|
+
|
|
2503
|
+
// node_modules/drizzle-orm/column-builder.js
|
|
2504
|
+
class ColumnBuilder {
|
|
2505
|
+
static [entityKind] = "ColumnBuilder";
|
|
2506
|
+
config;
|
|
2507
|
+
constructor(name, dataType, columnType) {
|
|
2508
|
+
this.config = {
|
|
2509
|
+
name,
|
|
2510
|
+
keyAsName: name === "",
|
|
2511
|
+
notNull: false,
|
|
2512
|
+
default: undefined,
|
|
2513
|
+
hasDefault: false,
|
|
2514
|
+
primaryKey: false,
|
|
2515
|
+
isUnique: false,
|
|
2516
|
+
uniqueName: undefined,
|
|
2517
|
+
uniqueType: undefined,
|
|
2518
|
+
dataType,
|
|
2519
|
+
columnType,
|
|
2520
|
+
generated: undefined
|
|
2521
|
+
};
|
|
2522
|
+
}
|
|
2523
|
+
$type() {
|
|
2524
|
+
return this;
|
|
2525
|
+
}
|
|
2526
|
+
notNull() {
|
|
2527
|
+
this.config.notNull = true;
|
|
2528
|
+
return this;
|
|
2529
|
+
}
|
|
2530
|
+
default(value) {
|
|
2531
|
+
this.config.default = value;
|
|
2532
|
+
this.config.hasDefault = true;
|
|
2533
|
+
return this;
|
|
2534
|
+
}
|
|
2535
|
+
$defaultFn(fn) {
|
|
2536
|
+
this.config.defaultFn = fn;
|
|
2537
|
+
this.config.hasDefault = true;
|
|
2538
|
+
return this;
|
|
2539
|
+
}
|
|
2540
|
+
$default = this.$defaultFn;
|
|
2541
|
+
$onUpdateFn(fn) {
|
|
2542
|
+
this.config.onUpdateFn = fn;
|
|
2543
|
+
this.config.hasDefault = true;
|
|
2544
|
+
return this;
|
|
2545
|
+
}
|
|
2546
|
+
$onUpdate = this.$onUpdateFn;
|
|
2547
|
+
primaryKey() {
|
|
2548
|
+
this.config.primaryKey = true;
|
|
2549
|
+
this.config.notNull = true;
|
|
2550
|
+
return this;
|
|
2551
|
+
}
|
|
2552
|
+
setName(name) {
|
|
2553
|
+
if (this.config.name !== "")
|
|
2554
|
+
return;
|
|
2555
|
+
this.config.name = name;
|
|
2556
|
+
}
|
|
2557
|
+
}
|
|
2558
|
+
|
|
2559
|
+
// node_modules/drizzle-orm/table.utils.js
|
|
2560
|
+
var TableName = Symbol.for("drizzle:Name");
|
|
2561
|
+
|
|
2562
|
+
// node_modules/drizzle-orm/tracing-utils.js
|
|
2563
|
+
function iife(fn, ...args) {
|
|
2564
|
+
return fn(...args);
|
|
2565
|
+
}
|
|
2566
|
+
|
|
2567
|
+
// node_modules/drizzle-orm/pg-core/unique-constraint.js
|
|
2568
|
+
function uniqueKeyName(table, columns) {
|
|
2569
|
+
return `${table[TableName]}_${columns.join("_")}_unique`;
|
|
2570
|
+
}
|
|
2571
|
+
|
|
2572
|
+
// node_modules/drizzle-orm/pg-core/columns/common.js
|
|
2244
2573
|
class PgColumn extends Column {
|
|
2245
2574
|
constructor(table, config) {
|
|
2246
2575
|
if (!config.uniqueName) {
|
|
@@ -3454,9 +3783,9 @@ function mapRelationalRow(tablesConfig, tableConfig, row, buildQueryResultSelect
|
|
|
3454
3783
|
|
|
3455
3784
|
// src/cli/index.ts
|
|
3456
3785
|
import { randomUUID } from "crypto";
|
|
3457
|
-
import { readFileSync as
|
|
3458
|
-
import { join as
|
|
3459
|
-
import { homedir as
|
|
3786
|
+
import { readFileSync as readFileSync3, existsSync as existsSync3, mkdirSync as mkdirSync3, writeFileSync as writeFileSync2 } from "fs";
|
|
3787
|
+
import { join as join7 } from "path";
|
|
3788
|
+
import { homedir as homedir7 } from "os";
|
|
3460
3789
|
|
|
3461
3790
|
// node_modules/drizzle-orm/bun-sqlite/driver.js
|
|
3462
3791
|
import { Database } from "bun:sqlite";
|
|
@@ -5739,10 +6068,48 @@ function drizzle(...params) {
|
|
|
5739
6068
|
drizzle2.mock = mock;
|
|
5740
6069
|
})(drizzle || (drizzle = {}));
|
|
5741
6070
|
|
|
6071
|
+
// node_modules/drizzle-orm/migrator.js
|
|
6072
|
+
import crypto from "crypto";
|
|
6073
|
+
import fs from "fs";
|
|
6074
|
+
function readMigrationFiles(config) {
|
|
6075
|
+
const migrationFolderTo = config.migrationsFolder;
|
|
6076
|
+
const migrationQueries = [];
|
|
6077
|
+
const journalPath = `${migrationFolderTo}/meta/_journal.json`;
|
|
6078
|
+
if (!fs.existsSync(journalPath)) {
|
|
6079
|
+
throw new Error(`Can't find meta/_journal.json file`);
|
|
6080
|
+
}
|
|
6081
|
+
const journalAsString = fs.readFileSync(`${migrationFolderTo}/meta/_journal.json`).toString();
|
|
6082
|
+
const journal = JSON.parse(journalAsString);
|
|
6083
|
+
for (const journalEntry of journal.entries) {
|
|
6084
|
+
const migrationPath = `${migrationFolderTo}/${journalEntry.tag}.sql`;
|
|
6085
|
+
try {
|
|
6086
|
+
const query = fs.readFileSync(`${migrationFolderTo}/${journalEntry.tag}.sql`).toString();
|
|
6087
|
+
const result = query.split("--> statement-breakpoint").map((it) => {
|
|
6088
|
+
return it;
|
|
6089
|
+
});
|
|
6090
|
+
migrationQueries.push({
|
|
6091
|
+
sql: result,
|
|
6092
|
+
bps: journalEntry.breakpoints,
|
|
6093
|
+
folderMillis: journalEntry.when,
|
|
6094
|
+
hash: crypto.createHash("sha256").update(query).digest("hex")
|
|
6095
|
+
});
|
|
6096
|
+
} catch {
|
|
6097
|
+
throw new Error(`No file ${migrationPath} found in ${migrationFolderTo} folder`);
|
|
6098
|
+
}
|
|
6099
|
+
}
|
|
6100
|
+
return migrationQueries;
|
|
6101
|
+
}
|
|
6102
|
+
|
|
6103
|
+
// node_modules/drizzle-orm/bun-sqlite/migrator.js
|
|
6104
|
+
function migrate(db, config) {
|
|
6105
|
+
const migrations = readMigrationFiles(config);
|
|
6106
|
+
db.dialect.migrate(migrations, db.session, config);
|
|
6107
|
+
}
|
|
6108
|
+
|
|
5742
6109
|
// src/db/index.ts
|
|
5743
6110
|
import { Database as Database2 } from "bun:sqlite";
|
|
5744
6111
|
import { mkdirSync } from "fs";
|
|
5745
|
-
import { dirname, join } from "path";
|
|
6112
|
+
import { dirname, join, resolve } from "path";
|
|
5746
6113
|
import { homedir } from "os";
|
|
5747
6114
|
|
|
5748
6115
|
// src/db/schema.ts
|
|
@@ -5794,52 +6161,55 @@ var DEFAULT_DB_PATH = join(homedir(), ".brains", "brains.db");
|
|
|
5794
6161
|
function ensureDir(filePath) {
|
|
5795
6162
|
mkdirSync(dirname(filePath), { recursive: true });
|
|
5796
6163
|
}
|
|
5797
|
-
function createTables(sqlite) {
|
|
5798
|
-
sqlite.exec(`
|
|
5799
|
-
CREATE TABLE IF NOT EXISTS fine_tuned_models (
|
|
5800
|
-
id TEXT PRIMARY KEY,
|
|
5801
|
-
base_model TEXT NOT NULL,
|
|
5802
|
-
name TEXT NOT NULL,
|
|
5803
|
-
provider TEXT NOT NULL,
|
|
5804
|
-
status TEXT NOT NULL DEFAULT 'pending',
|
|
5805
|
-
fine_tune_job_id TEXT,
|
|
5806
|
-
display_name TEXT,
|
|
5807
|
-
description TEXT,
|
|
5808
|
-
collection TEXT,
|
|
5809
|
-
tags TEXT,
|
|
5810
|
-
created_at INTEGER NOT NULL,
|
|
5811
|
-
updated_at INTEGER NOT NULL
|
|
5812
|
-
);
|
|
5813
|
-
|
|
5814
|
-
CREATE TABLE IF NOT EXISTS training_jobs (
|
|
5815
|
-
id TEXT PRIMARY KEY,
|
|
5816
|
-
model_id TEXT NOT NULL REFERENCES fine_tuned_models(id),
|
|
5817
|
-
provider TEXT NOT NULL,
|
|
5818
|
-
status TEXT NOT NULL,
|
|
5819
|
-
started_at INTEGER NOT NULL,
|
|
5820
|
-
finished_at INTEGER,
|
|
5821
|
-
metrics TEXT,
|
|
5822
|
-
error TEXT
|
|
5823
|
-
);
|
|
5824
|
-
|
|
5825
|
-
CREATE TABLE IF NOT EXISTS training_datasets (
|
|
5826
|
-
id TEXT PRIMARY KEY,
|
|
5827
|
-
source TEXT NOT NULL,
|
|
5828
|
-
file_path TEXT NOT NULL,
|
|
5829
|
-
example_count INTEGER NOT NULL,
|
|
5830
|
-
created_at INTEGER NOT NULL,
|
|
5831
|
-
used_in_job_id TEXT REFERENCES training_jobs(id)
|
|
5832
|
-
);
|
|
5833
|
-
`);
|
|
5834
|
-
}
|
|
5835
6164
|
function getDb(dbPath) {
|
|
5836
6165
|
const resolvedPath = dbPath ?? DEFAULT_DB_PATH;
|
|
5837
6166
|
ensureDir(resolvedPath);
|
|
5838
6167
|
const sqlite = new Database2(resolvedPath);
|
|
5839
6168
|
sqlite.run("PRAGMA journal_mode = WAL");
|
|
5840
6169
|
sqlite.run("PRAGMA foreign_keys = ON");
|
|
5841
|
-
|
|
5842
|
-
|
|
6170
|
+
const db = drizzle(sqlite, { schema: exports_schema });
|
|
6171
|
+
try {
|
|
6172
|
+
const migrationsFolder = resolve(import.meta.dir, "../../drizzle");
|
|
6173
|
+
migrate(db, { migrationsFolder });
|
|
6174
|
+
} catch {
|
|
6175
|
+
sqlite.exec(`
|
|
6176
|
+
CREATE TABLE IF NOT EXISTS fine_tuned_models (
|
|
6177
|
+
id TEXT PRIMARY KEY,
|
|
6178
|
+
base_model TEXT NOT NULL,
|
|
6179
|
+
name TEXT NOT NULL,
|
|
6180
|
+
provider TEXT NOT NULL,
|
|
6181
|
+
status TEXT NOT NULL DEFAULT 'pending',
|
|
6182
|
+
fine_tune_job_id TEXT,
|
|
6183
|
+
display_name TEXT,
|
|
6184
|
+
description TEXT,
|
|
6185
|
+
collection TEXT,
|
|
6186
|
+
tags TEXT,
|
|
6187
|
+
created_at INTEGER NOT NULL,
|
|
6188
|
+
updated_at INTEGER NOT NULL
|
|
6189
|
+
);
|
|
6190
|
+
|
|
6191
|
+
CREATE TABLE IF NOT EXISTS training_jobs (
|
|
6192
|
+
id TEXT PRIMARY KEY,
|
|
6193
|
+
model_id TEXT NOT NULL REFERENCES fine_tuned_models(id),
|
|
6194
|
+
provider TEXT NOT NULL,
|
|
6195
|
+
status TEXT NOT NULL,
|
|
6196
|
+
started_at INTEGER NOT NULL,
|
|
6197
|
+
finished_at INTEGER,
|
|
6198
|
+
metrics TEXT,
|
|
6199
|
+
error TEXT
|
|
6200
|
+
);
|
|
6201
|
+
|
|
6202
|
+
CREATE TABLE IF NOT EXISTS training_datasets (
|
|
6203
|
+
id TEXT PRIMARY KEY,
|
|
6204
|
+
source TEXT NOT NULL,
|
|
6205
|
+
file_path TEXT NOT NULL,
|
|
6206
|
+
example_count INTEGER NOT NULL,
|
|
6207
|
+
created_at INTEGER NOT NULL,
|
|
6208
|
+
used_in_job_id TEXT REFERENCES training_jobs(id)
|
|
6209
|
+
);
|
|
6210
|
+
`);
|
|
6211
|
+
}
|
|
6212
|
+
return db;
|
|
5843
6213
|
}
|
|
5844
6214
|
|
|
5845
6215
|
// node_modules/openai/internal/qs/formats.mjs
|
|
@@ -6930,8 +7300,8 @@ function _addRequestID(value, response) {
|
|
|
6930
7300
|
|
|
6931
7301
|
class APIPromise extends Promise {
|
|
6932
7302
|
constructor(responsePromise, parseResponse = defaultParseResponse) {
|
|
6933
|
-
super((
|
|
6934
|
-
|
|
7303
|
+
super((resolve2) => {
|
|
7304
|
+
resolve2(null);
|
|
6935
7305
|
});
|
|
6936
7306
|
this.responsePromise = responsePromise;
|
|
6937
7307
|
this.parseResponse = parseResponse;
|
|
@@ -7442,7 +7812,7 @@ var startsWithSchemeRegexp = /^[a-z][a-z0-9+.-]*:/i;
|
|
|
7442
7812
|
var isAbsoluteURL = (url) => {
|
|
7443
7813
|
return startsWithSchemeRegexp.test(url);
|
|
7444
7814
|
};
|
|
7445
|
-
var sleep = (ms) => new Promise((
|
|
7815
|
+
var sleep = (ms) => new Promise((resolve2) => setTimeout(resolve2, ms));
|
|
7446
7816
|
var validatePositiveInteger = (name, n) => {
|
|
7447
7817
|
if (typeof n !== "number" || !Number.isInteger(n)) {
|
|
7448
7818
|
throw new OpenAIError(`${name} must be an integer`);
|
|
@@ -7807,12 +8177,12 @@ class EventStream {
|
|
|
7807
8177
|
_EventStream_errored.set(this, false);
|
|
7808
8178
|
_EventStream_aborted.set(this, false);
|
|
7809
8179
|
_EventStream_catchingPromiseCreated.set(this, false);
|
|
7810
|
-
__classPrivateFieldSet3(this, _EventStream_connectedPromise, new Promise((
|
|
7811
|
-
__classPrivateFieldSet3(this, _EventStream_resolveConnectedPromise,
|
|
8180
|
+
__classPrivateFieldSet3(this, _EventStream_connectedPromise, new Promise((resolve2, reject) => {
|
|
8181
|
+
__classPrivateFieldSet3(this, _EventStream_resolveConnectedPromise, resolve2, "f");
|
|
7812
8182
|
__classPrivateFieldSet3(this, _EventStream_rejectConnectedPromise, reject, "f");
|
|
7813
8183
|
}), "f");
|
|
7814
|
-
__classPrivateFieldSet3(this, _EventStream_endPromise, new Promise((
|
|
7815
|
-
__classPrivateFieldSet3(this, _EventStream_resolveEndPromise,
|
|
8184
|
+
__classPrivateFieldSet3(this, _EventStream_endPromise, new Promise((resolve2, reject) => {
|
|
8185
|
+
__classPrivateFieldSet3(this, _EventStream_resolveEndPromise, resolve2, "f");
|
|
7816
8186
|
__classPrivateFieldSet3(this, _EventStream_rejectEndPromise, reject, "f");
|
|
7817
8187
|
}), "f");
|
|
7818
8188
|
__classPrivateFieldGet3(this, _EventStream_connectedPromise, "f").catch(() => {});
|
|
@@ -7864,11 +8234,11 @@ class EventStream {
|
|
|
7864
8234
|
return this;
|
|
7865
8235
|
}
|
|
7866
8236
|
emitted(event) {
|
|
7867
|
-
return new Promise((
|
|
8237
|
+
return new Promise((resolve2, reject) => {
|
|
7868
8238
|
__classPrivateFieldSet3(this, _EventStream_catchingPromiseCreated, true, "f");
|
|
7869
8239
|
if (event !== "error")
|
|
7870
8240
|
this.once("error", reject);
|
|
7871
|
-
this.once(event,
|
|
8241
|
+
this.once(event, resolve2);
|
|
7872
8242
|
});
|
|
7873
8243
|
}
|
|
7874
8244
|
async done() {
|
|
@@ -8026,7 +8396,7 @@ class AssistantStream extends EventStream {
|
|
|
8026
8396
|
if (done) {
|
|
8027
8397
|
return { value: undefined, done: true };
|
|
8028
8398
|
}
|
|
8029
|
-
return new Promise((
|
|
8399
|
+
return new Promise((resolve2, reject) => readQueue.push({ resolve: resolve2, reject })).then((chunk2) => chunk2 ? { value: chunk2, done: false } : { value: undefined, done: true });
|
|
8030
8400
|
}
|
|
8031
8401
|
const chunk = pushQueue.shift();
|
|
8032
8402
|
return { value: chunk, done: false };
|
|
@@ -9306,11 +9676,11 @@ class ChatCompletionStream extends AbstractChatCompletionRunner {
|
|
|
9306
9676
|
}
|
|
9307
9677
|
return this._addChatCompletion(__classPrivateFieldGet6(this, _ChatCompletionStream_instances, "m", _ChatCompletionStream_endRequest).call(this));
|
|
9308
9678
|
}
|
|
9309
|
-
[(_ChatCompletionStream_params = new WeakMap, _ChatCompletionStream_choiceEventStates = new WeakMap, _ChatCompletionStream_currentChatCompletionSnapshot = new WeakMap, _ChatCompletionStream_instances = new WeakSet, _ChatCompletionStream_beginRequest = function
|
|
9679
|
+
[(_ChatCompletionStream_params = new WeakMap, _ChatCompletionStream_choiceEventStates = new WeakMap, _ChatCompletionStream_currentChatCompletionSnapshot = new WeakMap, _ChatCompletionStream_instances = new WeakSet, _ChatCompletionStream_beginRequest = function _ChatCompletionStream_beginRequest() {
|
|
9310
9680
|
if (this.ended)
|
|
9311
9681
|
return;
|
|
9312
9682
|
__classPrivateFieldSet5(this, _ChatCompletionStream_currentChatCompletionSnapshot, undefined, "f");
|
|
9313
|
-
}, _ChatCompletionStream_getChoiceEventState = function
|
|
9683
|
+
}, _ChatCompletionStream_getChoiceEventState = function _ChatCompletionStream_getChoiceEventState(choice) {
|
|
9314
9684
|
let state = __classPrivateFieldGet6(this, _ChatCompletionStream_choiceEventStates, "f")[choice.index];
|
|
9315
9685
|
if (state) {
|
|
9316
9686
|
return state;
|
|
@@ -9325,7 +9695,7 @@ class ChatCompletionStream extends AbstractChatCompletionRunner {
|
|
|
9325
9695
|
};
|
|
9326
9696
|
__classPrivateFieldGet6(this, _ChatCompletionStream_choiceEventStates, "f")[choice.index] = state;
|
|
9327
9697
|
return state;
|
|
9328
|
-
}, _ChatCompletionStream_addChunk = function
|
|
9698
|
+
}, _ChatCompletionStream_addChunk = function _ChatCompletionStream_addChunk(chunk) {
|
|
9329
9699
|
if (this.ended)
|
|
9330
9700
|
return;
|
|
9331
9701
|
const completion = __classPrivateFieldGet6(this, _ChatCompletionStream_instances, "m", _ChatCompletionStream_accumulateChatCompletion).call(this, chunk);
|
|
@@ -9392,7 +9762,7 @@ class ChatCompletionStream extends AbstractChatCompletionRunner {
|
|
|
9392
9762
|
}
|
|
9393
9763
|
}
|
|
9394
9764
|
}
|
|
9395
|
-
}, _ChatCompletionStream_emitToolCallDoneEvent = function
|
|
9765
|
+
}, _ChatCompletionStream_emitToolCallDoneEvent = function _ChatCompletionStream_emitToolCallDoneEvent(choiceSnapshot, toolCallIndex) {
|
|
9396
9766
|
const state = __classPrivateFieldGet6(this, _ChatCompletionStream_instances, "m", _ChatCompletionStream_getChoiceEventState).call(this, choiceSnapshot);
|
|
9397
9767
|
if (state.done_tool_calls.has(toolCallIndex)) {
|
|
9398
9768
|
return;
|
|
@@ -9415,7 +9785,7 @@ class ChatCompletionStream extends AbstractChatCompletionRunner {
|
|
|
9415
9785
|
} else {
|
|
9416
9786
|
assertNever2(toolCallSnapshot.type);
|
|
9417
9787
|
}
|
|
9418
|
-
}, _ChatCompletionStream_emitContentDoneEvents = function
|
|
9788
|
+
}, _ChatCompletionStream_emitContentDoneEvents = function _ChatCompletionStream_emitContentDoneEvents(choiceSnapshot) {
|
|
9419
9789
|
const state = __classPrivateFieldGet6(this, _ChatCompletionStream_instances, "m", _ChatCompletionStream_getChoiceEventState).call(this, choiceSnapshot);
|
|
9420
9790
|
if (choiceSnapshot.message.content && !state.content_done) {
|
|
9421
9791
|
state.content_done = true;
|
|
@@ -9437,7 +9807,7 @@ class ChatCompletionStream extends AbstractChatCompletionRunner {
|
|
|
9437
9807
|
state.logprobs_refusal_done = true;
|
|
9438
9808
|
this._emit("logprobs.refusal.done", { refusal: choiceSnapshot.logprobs.refusal });
|
|
9439
9809
|
}
|
|
9440
|
-
}, _ChatCompletionStream_endRequest = function
|
|
9810
|
+
}, _ChatCompletionStream_endRequest = function _ChatCompletionStream_endRequest() {
|
|
9441
9811
|
if (this.ended) {
|
|
9442
9812
|
throw new OpenAIError(`stream has ended, this shouldn't happen`);
|
|
9443
9813
|
}
|
|
@@ -9448,13 +9818,13 @@ class ChatCompletionStream extends AbstractChatCompletionRunner {
|
|
|
9448
9818
|
__classPrivateFieldSet5(this, _ChatCompletionStream_currentChatCompletionSnapshot, undefined, "f");
|
|
9449
9819
|
__classPrivateFieldSet5(this, _ChatCompletionStream_choiceEventStates, [], "f");
|
|
9450
9820
|
return finalizeChatCompletion(snapshot, __classPrivateFieldGet6(this, _ChatCompletionStream_params, "f"));
|
|
9451
|
-
}, _ChatCompletionStream_getAutoParseableResponseFormat = function
|
|
9821
|
+
}, _ChatCompletionStream_getAutoParseableResponseFormat = function _ChatCompletionStream_getAutoParseableResponseFormat() {
|
|
9452
9822
|
const responseFormat = __classPrivateFieldGet6(this, _ChatCompletionStream_params, "f")?.response_format;
|
|
9453
9823
|
if (isAutoParsableResponseFormat(responseFormat)) {
|
|
9454
9824
|
return responseFormat;
|
|
9455
9825
|
}
|
|
9456
9826
|
return null;
|
|
9457
|
-
}, _ChatCompletionStream_accumulateChatCompletion = function
|
|
9827
|
+
}, _ChatCompletionStream_accumulateChatCompletion = function _ChatCompletionStream_accumulateChatCompletion(chunk) {
|
|
9458
9828
|
var _a, _b, _c, _d;
|
|
9459
9829
|
let snapshot = __classPrivateFieldGet6(this, _ChatCompletionStream_currentChatCompletionSnapshot, "f");
|
|
9460
9830
|
const { choices, ...rest } = chunk;
|
|
@@ -9591,7 +9961,7 @@ class ChatCompletionStream extends AbstractChatCompletionRunner {
|
|
|
9591
9961
|
if (done) {
|
|
9592
9962
|
return { value: undefined, done: true };
|
|
9593
9963
|
}
|
|
9594
|
-
return new Promise((
|
|
9964
|
+
return new Promise((resolve2, reject) => readQueue.push({ resolve: resolve2, reject })).then((chunk2) => chunk2 ? { value: chunk2, done: false } : { value: undefined, done: true });
|
|
9595
9965
|
}
|
|
9596
9966
|
const chunk = pushQueue.shift();
|
|
9597
9967
|
return { value: chunk, done: false };
|
|
@@ -10680,11 +11050,11 @@ class ResponseStream extends EventStream {
|
|
|
10680
11050
|
}
|
|
10681
11051
|
return __classPrivateFieldGet7(this, _ResponseStream_instances, "m", _ResponseStream_endRequest).call(this);
|
|
10682
11052
|
}
|
|
10683
|
-
[(_ResponseStream_params = new WeakMap, _ResponseStream_currentResponseSnapshot = new WeakMap, _ResponseStream_finalResponse = new WeakMap, _ResponseStream_instances = new WeakSet, _ResponseStream_beginRequest = function
|
|
11053
|
+
[(_ResponseStream_params = new WeakMap, _ResponseStream_currentResponseSnapshot = new WeakMap, _ResponseStream_finalResponse = new WeakMap, _ResponseStream_instances = new WeakSet, _ResponseStream_beginRequest = function _ResponseStream_beginRequest() {
|
|
10684
11054
|
if (this.ended)
|
|
10685
11055
|
return;
|
|
10686
11056
|
__classPrivateFieldSet6(this, _ResponseStream_currentResponseSnapshot, undefined, "f");
|
|
10687
|
-
}, _ResponseStream_addEvent = function
|
|
11057
|
+
}, _ResponseStream_addEvent = function _ResponseStream_addEvent(event, starting_after) {
|
|
10688
11058
|
if (this.ended)
|
|
10689
11059
|
return;
|
|
10690
11060
|
const maybeEmit = (name, event2) => {
|
|
@@ -10732,7 +11102,7 @@ class ResponseStream extends EventStream {
|
|
|
10732
11102
|
maybeEmit(event.type, event);
|
|
10733
11103
|
break;
|
|
10734
11104
|
}
|
|
10735
|
-
}, _ResponseStream_endRequest = function
|
|
11105
|
+
}, _ResponseStream_endRequest = function _ResponseStream_endRequest() {
|
|
10736
11106
|
if (this.ended) {
|
|
10737
11107
|
throw new OpenAIError(`stream has ended, this shouldn't happen`);
|
|
10738
11108
|
}
|
|
@@ -10744,7 +11114,7 @@ class ResponseStream extends EventStream {
|
|
|
10744
11114
|
const parsedResponse = finalizeResponse(snapshot, __classPrivateFieldGet7(this, _ResponseStream_params, "f"));
|
|
10745
11115
|
__classPrivateFieldSet6(this, _ResponseStream_finalResponse, parsedResponse, "f");
|
|
10746
11116
|
return parsedResponse;
|
|
10747
|
-
}, _ResponseStream_accumulateResponse = function
|
|
11117
|
+
}, _ResponseStream_accumulateResponse = function _ResponseStream_accumulateResponse(event) {
|
|
10748
11118
|
let snapshot = __classPrivateFieldGet7(this, _ResponseStream_currentResponseSnapshot, "f");
|
|
10749
11119
|
if (!snapshot) {
|
|
10750
11120
|
if (event.type !== "response.created") {
|
|
@@ -10840,7 +11210,7 @@ class ResponseStream extends EventStream {
|
|
|
10840
11210
|
if (done) {
|
|
10841
11211
|
return { value: undefined, done: true };
|
|
10842
11212
|
}
|
|
10843
|
-
return new Promise((
|
|
11213
|
+
return new Promise((resolve2, reject) => readQueue.push({ resolve: resolve2, reject })).then((event2) => event2 ? { value: event2, done: false } : { value: undefined, done: true });
|
|
10844
11214
|
}
|
|
10845
11215
|
const event = pushQueue.shift();
|
|
10846
11216
|
return { value: event, done: false };
|
|
@@ -11327,558 +11697,307 @@ var _deployments_endpoints = new Set([
|
|
|
11327
11697
|
var openai_default = OpenAI;
|
|
11328
11698
|
|
|
11329
11699
|
// src/lib/providers/openai.ts
|
|
11330
|
-
import { readFileSync } from "fs";
|
|
11331
|
-
function getClient() {
|
|
11332
|
-
const apiKey = process.env.OPENAI_API_KEY;
|
|
11333
|
-
if (!apiKey) {
|
|
11334
|
-
throw new Error("OPENAI_API_KEY environment variable is required");
|
|
11335
|
-
}
|
|
11336
|
-
return new openai_default({ apiKey });
|
|
11337
|
-
}
|
|
11338
|
-
async function uploadTrainingFile(filePath) {
|
|
11339
|
-
const client = getClient();
|
|
11340
|
-
const fileContent = readFileSync(filePath);
|
|
11341
|
-
const blob2 = new Blob([fileContent], { type: "application/jsonl" });
|
|
11342
|
-
const file = new File([blob2], filePath.split("/").pop() ?? "training.jsonl", { type: "application/jsonl" });
|
|
11343
|
-
const response = await client.files.create({
|
|
11344
|
-
file,
|
|
11345
|
-
purpose: "fine-tune"
|
|
11346
|
-
});
|
|
11347
|
-
return { fileId: response.id };
|
|
11348
|
-
}
|
|
11349
|
-
async function createFineTuneJob(fileId, baseModel, suffix) {
|
|
11350
|
-
const client = getClient();
|
|
11351
|
-
const params = {
|
|
11352
|
-
training_file: fileId,
|
|
11353
|
-
model: baseModel
|
|
11354
|
-
};
|
|
11355
|
-
if (suffix) {
|
|
11356
|
-
params.suffix = suffix;
|
|
11357
|
-
}
|
|
11358
|
-
const response = await client.fineTuning.jobs.create(params);
|
|
11359
|
-
return { jobId: response.id, status: response.status };
|
|
11360
|
-
}
|
|
11361
|
-
async function getFineTuneStatus(jobId) {
|
|
11362
|
-
const client = getClient();
|
|
11363
|
-
const response = await client.fineTuning.jobs.retrieve(jobId);
|
|
11364
|
-
return {
|
|
11365
|
-
jobId: response.id,
|
|
11366
|
-
status: response.status,
|
|
11367
|
-
fineTunedModel: response.fine_tuned_model ?? undefined,
|
|
11368
|
-
error: response.error?.message ?? undefined
|
|
11369
|
-
};
|
|
11370
|
-
}
|
|
11371
|
-
async function listFineTunedModels() {
|
|
11372
|
-
const client = getClient();
|
|
11373
|
-
const jobs = await client.fineTuning.jobs.list();
|
|
11374
|
-
return jobs.data.map((job) => ({
|
|
11375
|
-
id: job.id,
|
|
11376
|
-
model: job.fine_tuned_model ?? job.model,
|
|
11377
|
-
status: job.status,
|
|
11378
|
-
created: job.created_at
|
|
11379
|
-
}));
|
|
11380
|
-
}
|
|
11381
|
-
|
|
11382
|
-
// src/lib/providers/thinker-labs.ts
|
|
11383
|
-
var DEFAULT_BASE_URL = "https://api.thinkerlabs.ai/v1";
|
|
11384
|
-
function getConfig() {
|
|
11385
|
-
const apiKey = process.env.THINKER_LABS_API_KEY;
|
|
11386
|
-
const baseUrl = process.env.THINKER_LABS_BASE_URL ?? DEFAULT_BASE_URL;
|
|
11387
|
-
if (!apiKey)
|
|
11388
|
-
throw new Error("THINKER_LABS_API_KEY environment variable is required");
|
|
11389
|
-
return { apiKey, baseUrl };
|
|
11390
|
-
}
|
|
11391
|
-
async function request(method, path, body, file) {
|
|
11392
|
-
const { apiKey, baseUrl } = getConfig();
|
|
11393
|
-
const headers = { Authorization: `Bearer ${apiKey}` };
|
|
11394
|
-
let fetchBody;
|
|
11395
|
-
if (file) {
|
|
11396
|
-
const form = new FormData;
|
|
11397
|
-
form.append("file", new Blob([file.data], { type: "text/plain" }), file.name);
|
|
11398
|
-
if (body) {
|
|
11399
|
-
for (const [k, v] of Object.entries(body)) {
|
|
11400
|
-
form.append(k, v);
|
|
11401
|
-
}
|
|
11402
|
-
}
|
|
11403
|
-
fetchBody = form;
|
|
11404
|
-
} else if (body) {
|
|
11405
|
-
headers["Content-Type"] = "application/json";
|
|
11406
|
-
fetchBody = JSON.stringify(body);
|
|
11407
|
-
}
|
|
11408
|
-
const res = await fetch(`${baseUrl}${path}`, { method, headers, body: fetchBody });
|
|
11409
|
-
if (!res.ok) {
|
|
11410
|
-
const text2 = await res.text();
|
|
11411
|
-
throw new Error(`Thinker Labs API error ${res.status}: ${text2}`);
|
|
11412
|
-
}
|
|
11413
|
-
if (res.status === 204)
|
|
11414
|
-
return;
|
|
11415
|
-
return res.json();
|
|
11416
|
-
}
|
|
11417
|
-
async function uploadTrainingData(filePath) {
|
|
11418
|
-
const fileContent = await Bun.file(filePath).text();
|
|
11419
|
-
const fileName = filePath.split("/").pop() ?? "training.jsonl";
|
|
11420
|
-
const result = await request("POST", "/datasets/upload", undefined, { name: fileName, data: fileContent });
|
|
11421
|
-
return { datasetId: result.id };
|
|
11422
|
-
}
|
|
11423
|
-
async function startFineTune(datasetId, baseModel, name) {
|
|
11424
|
-
const result = await request("POST", "/fine-tunes", {
|
|
11425
|
-
dataset_id: datasetId,
|
|
11426
|
-
base_model: baseModel,
|
|
11427
|
-
...name ? { name } : {}
|
|
11428
|
-
});
|
|
11429
|
-
return { jobId: result.id, status: result.status };
|
|
11430
|
-
}
|
|
11431
|
-
async function getStatus(jobId) {
|
|
11432
|
-
const result = await request("GET", `/fine-tunes/${jobId}`);
|
|
11433
|
-
return {
|
|
11434
|
-
jobId: result.id,
|
|
11435
|
-
status: result.status,
|
|
11436
|
-
modelId: result.model_id,
|
|
11437
|
-
error: result.error
|
|
11438
|
-
};
|
|
11439
|
-
}
|
|
11440
|
-
async function listModels() {
|
|
11441
|
-
const result = await request("GET", "/fine-tunes");
|
|
11442
|
-
return (result.data ?? []).map((m) => ({
|
|
11443
|
-
id: m.id,
|
|
11444
|
-
name: m.name,
|
|
11445
|
-
status: m.status,
|
|
11446
|
-
baseModel: m.base_model,
|
|
11447
|
-
createdAt: m.created_at
|
|
11448
|
-
}));
|
|
11449
|
-
}
|
|
11450
|
-
async function cancelJob(jobId) {
|
|
11451
|
-
await request("DELETE", `/fine-tunes/${jobId}`);
|
|
11452
|
-
}
|
|
11453
|
-
|
|
11454
|
-
class ThinkerLabsProvider {
|
|
11455
|
-
uploadTrainingData = uploadTrainingData;
|
|
11456
|
-
startFineTune = startFineTune;
|
|
11457
|
-
getStatus = getStatus;
|
|
11458
|
-
listModels = listModels;
|
|
11459
|
-
cancelJob = cancelJob;
|
|
11460
|
-
async uploadTrainingFile(filePath) {
|
|
11461
|
-
const { datasetId } = await uploadTrainingData(filePath);
|
|
11462
|
-
return { fileId: datasetId };
|
|
11463
|
-
}
|
|
11464
|
-
async createFineTuneJob(fileId, baseModel, suffix) {
|
|
11465
|
-
return startFineTune(fileId, baseModel, suffix);
|
|
11466
|
-
}
|
|
11467
|
-
async getFineTuneStatus(jobId) {
|
|
11468
|
-
const result = await getStatus(jobId);
|
|
11469
|
-
return { jobId: result.jobId, status: result.status, fineTunedModel: result.modelId, error: result.error };
|
|
11470
|
-
}
|
|
11471
|
-
async listFineTunedModels() {
|
|
11472
|
-
const models = await listModels();
|
|
11473
|
-
return models.map((m) => ({ id: m.id, model: m.baseModel, status: m.status, created: m.createdAt }));
|
|
11474
|
-
}
|
|
11475
|
-
async cancelFineTuneJob(jobId) {
|
|
11476
|
-
return cancelJob(jobId);
|
|
11477
|
-
}
|
|
11478
|
-
}
|
|
11479
|
-
|
|
11480
|
-
// src/cli/ui.ts
|
|
11481
|
-
import chalk from "chalk";
|
|
11482
|
-
function printTable(headers, rows) {
|
|
11483
|
-
if (rows.length === 0) {
|
|
11484
|
-
console.log(chalk.dim(" (no records)"));
|
|
11485
|
-
return;
|
|
11486
|
-
}
|
|
11487
|
-
const colWidths = headers.map((h, i) => Math.max(h.length, ...rows.map((r) => (r[i] ?? "").length)));
|
|
11488
|
-
const separator = colWidths.map((w) => "\u2500".repeat(w + 2)).join("\u253C");
|
|
11489
|
-
const headerLine = headers.map((h, i) => ` ${chalk.bold(h.padEnd(colWidths[i] ?? 0))} `).join("\u2502");
|
|
11490
|
-
const topBorder = colWidths.map((w) => "\u2500".repeat(w + 2)).join("\u252C");
|
|
11491
|
-
const midBorder = separator;
|
|
11492
|
-
const bottomBorder = colWidths.map((w) => "\u2500".repeat(w + 2)).join("\u2534");
|
|
11493
|
-
console.log("\u250C" + topBorder + "\u2510");
|
|
11494
|
-
console.log("\u2502" + headerLine + "\u2502");
|
|
11495
|
-
console.log("\u251C" + midBorder + "\u2524");
|
|
11496
|
-
for (const row of rows) {
|
|
11497
|
-
const rowLine = row.map((cell, i) => ` ${(cell ?? "").padEnd(colWidths[i] ?? 0)} `).join("\u2502");
|
|
11498
|
-
console.log("\u2502" + rowLine + "\u2502");
|
|
11499
|
-
}
|
|
11500
|
-
console.log("\u2514" + bottomBorder + "\u2518");
|
|
11501
|
-
}
|
|
11502
|
-
var STATUS_COLORS = {
|
|
11503
|
-
succeeded: chalk.green,
|
|
11504
|
-
running: chalk.cyan,
|
|
11505
|
-
pending: chalk.yellow,
|
|
11506
|
-
failed: chalk.red,
|
|
11507
|
-
cancelled: chalk.dim,
|
|
11508
|
-
queued: chalk.yellow,
|
|
11509
|
-
validating_files: chalk.blue
|
|
11510
|
-
};
|
|
11511
|
-
function printStatus(status) {
|
|
11512
|
-
const colorFn = STATUS_COLORS[status] ?? chalk.white;
|
|
11513
|
-
return colorFn(`\u25CF ${status}`);
|
|
11514
|
-
}
|
|
11515
|
-
function printJson(obj) {
|
|
11516
|
-
console.log(JSON.stringify(obj, null, 2));
|
|
11517
|
-
}
|
|
11518
|
-
function printError(message) {
|
|
11519
|
-
console.error(chalk.red("\u2717 Error: ") + message);
|
|
11520
|
-
}
|
|
11521
|
-
function printSuccess(message) {
|
|
11522
|
-
console.log(chalk.green("\u2713 ") + message);
|
|
11523
|
-
}
|
|
11524
|
-
function printInfo(message) {
|
|
11525
|
-
console.log(chalk.dim(" " + message));
|
|
11526
|
-
}
|
|
11527
|
-
|
|
11528
|
-
// src/lib/gatherers/todos.ts
|
|
11529
|
-
import { Database as Database3 } from "bun:sqlite";
|
|
11530
|
-
import { homedir as homedir2 } from "os";
|
|
11531
|
-
import { join as join2 } from "path";
|
|
11532
|
-
var SYSTEM_PROMPT = "You are a task management assistant that helps users create, update, search, and manage tasks and projects.";
|
|
11533
|
-
function taskToCreateExample(task) {
|
|
11534
|
-
const userMsg = `Create a task: ${task.title}${task.description ? `
|
|
11700
|
+
import { readFileSync as readFileSync2 } from "fs";
|
|
11535
11701
|
|
|
11536
|
-
|
|
11537
|
-
|
|
11538
|
-
|
|
11539
|
-
|
|
11540
|
-
|
|
11541
|
-
|
|
11542
|
-
|
|
11543
|
-
|
|
11544
|
-
|
|
11545
|
-
|
|
11546
|
-
|
|
11547
|
-
|
|
11548
|
-
|
|
11549
|
-
|
|
11550
|
-
{ role: "assistant", content: `Created task: ${JSON.stringify(taskDetails, null, 2)}` }
|
|
11551
|
-
]
|
|
11552
|
-
};
|
|
11553
|
-
}
|
|
11554
|
-
function taskToStatusUpdateExample(task) {
|
|
11555
|
-
if (!task.completed_at && task.status === "pending")
|
|
11556
|
-
return null;
|
|
11557
|
-
const id = task.short_id ?? task.id;
|
|
11558
|
-
return {
|
|
11559
|
-
messages: [
|
|
11560
|
-
{ role: "system", content: SYSTEM_PROMPT },
|
|
11561
|
-
{ role: "user", content: `Mark task ${id} as ${task.status}` },
|
|
11562
|
-
{ role: "assistant", content: `Task ${id} has been updated to status: ${task.status}. ${task.completed_at ? `Completed at: ${task.completed_at}` : ""}`.trim() }
|
|
11563
|
-
]
|
|
11564
|
-
};
|
|
11702
|
+
// src/lib/config.ts
|
|
11703
|
+
import { readFileSync, writeFileSync, mkdirSync as mkdirSync2, existsSync } from "fs";
|
|
11704
|
+
import { join as join2, dirname as dirname2 } from "path";
|
|
11705
|
+
import { homedir as homedir2 } from "os";
|
|
11706
|
+
var CONFIG_KEYS = ["OPENAI_API_KEY", "THINKER_LABS_API_KEY", "THINKER_LABS_BASE_URL"];
|
|
11707
|
+
var CONFIG_PATH = join2(homedir2(), ".brains", "config.json");
|
|
11708
|
+
function readConfigFile() {
|
|
11709
|
+
if (!existsSync(CONFIG_PATH))
|
|
11710
|
+
return {};
|
|
11711
|
+
try {
|
|
11712
|
+
return JSON.parse(readFileSync(CONFIG_PATH, "utf-8"));
|
|
11713
|
+
} catch {
|
|
11714
|
+
return {};
|
|
11715
|
+
}
|
|
11565
11716
|
}
|
|
11566
|
-
function
|
|
11567
|
-
|
|
11568
|
-
|
|
11569
|
-
|
|
11570
|
-
|
|
11571
|
-
|
|
11572
|
-
|
|
11573
|
-
|
|
11574
|
-
|
|
11575
|
-
|
|
11576
|
-
|
|
11577
|
-
|
|
11578
|
-
|
|
11579
|
-
|
|
11717
|
+
function writeConfigFile(data) {
|
|
11718
|
+
mkdirSync2(dirname2(CONFIG_PATH), { recursive: true });
|
|
11719
|
+
writeFileSync(CONFIG_PATH, JSON.stringify(data, null, 2) + `
|
|
11720
|
+
`, "utf-8");
|
|
11721
|
+
}
|
|
11722
|
+
function getConfigValue(key) {
|
|
11723
|
+
if (process.env[key])
|
|
11724
|
+
return process.env[key];
|
|
11725
|
+
return readConfigFile()[key];
|
|
11726
|
+
}
|
|
11727
|
+
function setConfigValue(key, value) {
|
|
11728
|
+
const data = readConfigFile();
|
|
11729
|
+
data[key] = value;
|
|
11730
|
+
writeConfigFile(data);
|
|
11731
|
+
}
|
|
11732
|
+
function listConfig() {
|
|
11733
|
+
const file = readConfigFile();
|
|
11734
|
+
return CONFIG_KEYS.map((key) => {
|
|
11735
|
+
if (process.env[key])
|
|
11736
|
+
return { key, value: process.env[key], source: "env" };
|
|
11737
|
+
if (file[key])
|
|
11738
|
+
return { key, value: file[key], source: "file" };
|
|
11739
|
+
return { key, value: "", source: "unset" };
|
|
11740
|
+
});
|
|
11580
11741
|
}
|
|
11581
|
-
|
|
11582
|
-
const
|
|
11583
|
-
|
|
11584
|
-
|
|
11585
|
-
|
|
11586
|
-
|
|
11587
|
-
|
|
11588
|
-
|
|
11589
|
-
|
|
11742
|
+
function deleteConfigValue(key) {
|
|
11743
|
+
const data = readConfigFile();
|
|
11744
|
+
delete data[key];
|
|
11745
|
+
writeConfigFile(data);
|
|
11746
|
+
}
|
|
11747
|
+
|
|
11748
|
+
// src/lib/retry.ts
|
|
11749
|
+
var RETRYABLE_STATUS_CODES = new Set([429, 500, 502, 503, 504]);
|
|
11750
|
+
function isRetryableError(err) {
|
|
11751
|
+
if (err instanceof Error) {
|
|
11752
|
+
if (err.message.includes("fetch failed") || err.message.includes("ECONNRESET") || err.message.includes("ETIMEDOUT")) {
|
|
11753
|
+
return true;
|
|
11590
11754
|
}
|
|
11591
|
-
|
|
11592
|
-
if (
|
|
11593
|
-
|
|
11594
|
-
|
|
11755
|
+
const match = err.message.match(/^(\d{3})\s/);
|
|
11756
|
+
if (match) {
|
|
11757
|
+
const status = parseInt(match[1], 10);
|
|
11758
|
+
return RETRYABLE_STATUS_CODES.has(status);
|
|
11595
11759
|
}
|
|
11596
|
-
const
|
|
11597
|
-
|
|
11598
|
-
|
|
11599
|
-
|
|
11600
|
-
const statusEx = taskToStatusUpdateExample(task);
|
|
11601
|
-
if (statusEx)
|
|
11602
|
-
examples.push(statusEx);
|
|
11760
|
+
const tlMatch = err.message.match(/API error (\d{3})/);
|
|
11761
|
+
if (tlMatch) {
|
|
11762
|
+
const status = parseInt(tlMatch[1], 10);
|
|
11763
|
+
return RETRYABLE_STATUS_CODES.has(status);
|
|
11603
11764
|
}
|
|
11604
|
-
|
|
11605
|
-
|
|
11606
|
-
|
|
11765
|
+
}
|
|
11766
|
+
return false;
|
|
11767
|
+
}
|
|
11768
|
+
async function withRetry(fn, options = {}) {
|
|
11769
|
+
const { maxAttempts = 3, baseDelayMs = 1000, onRetry } = options;
|
|
11770
|
+
let lastError = new Error("Unknown error");
|
|
11771
|
+
for (let attempt = 1;attempt <= maxAttempts; attempt++) {
|
|
11772
|
+
try {
|
|
11773
|
+
return await fn();
|
|
11774
|
+
} catch (err) {
|
|
11775
|
+
lastError = err instanceof Error ? err : new Error(String(err));
|
|
11776
|
+
if (attempt === maxAttempts || !isRetryableError(lastError)) {
|
|
11777
|
+
throw lastError;
|
|
11778
|
+
}
|
|
11779
|
+
const delayMs = baseDelayMs * Math.pow(2, attempt - 1);
|
|
11780
|
+
onRetry?.(attempt, delayMs, lastError);
|
|
11781
|
+
await new Promise((resolve2) => setTimeout(resolve2, delayMs));
|
|
11607
11782
|
}
|
|
11608
|
-
const finalExamples = options.limit ? examples.slice(0, options.limit) : examples;
|
|
11609
|
-
return {
|
|
11610
|
-
source: "todos",
|
|
11611
|
-
examples: finalExamples,
|
|
11612
|
-
count: finalExamples.length
|
|
11613
|
-
};
|
|
11614
|
-
} finally {
|
|
11615
|
-
db.close();
|
|
11616
11783
|
}
|
|
11784
|
+
throw lastError;
|
|
11617
11785
|
}
|
|
11618
11786
|
|
|
11619
|
-
// src/lib/
|
|
11620
|
-
|
|
11621
|
-
|
|
11622
|
-
|
|
11623
|
-
|
|
11624
|
-
|
|
11625
|
-
return {
|
|
11626
|
-
messages: [
|
|
11627
|
-
{ role: "system", content: SYSTEM_PROMPT2 },
|
|
11628
|
-
{ role: "user", content: `What do you remember about "${memory.key}"?` },
|
|
11629
|
-
{
|
|
11630
|
-
role: "assistant",
|
|
11631
|
-
content: memory.summary ? `${memory.value}
|
|
11632
|
-
|
|
11633
|
-
Summary: ${memory.summary}` : memory.value
|
|
11634
|
-
}
|
|
11635
|
-
]
|
|
11636
|
-
};
|
|
11787
|
+
// src/lib/providers/openai.ts
|
|
11788
|
+
function getClient() {
|
|
11789
|
+
const apiKey = getConfigValue("OPENAI_API_KEY");
|
|
11790
|
+
if (!apiKey) {
|
|
11791
|
+
throw new Error("OPENAI_API_KEY is not set. Run: brains config set OPENAI_API_KEY <key>");
|
|
11792
|
+
}
|
|
11793
|
+
return new openai_default({ apiKey });
|
|
11637
11794
|
}
|
|
11638
|
-
function
|
|
11639
|
-
|
|
11640
|
-
|
|
11641
|
-
|
|
11642
|
-
|
|
11643
|
-
|
|
11644
|
-
|
|
11645
|
-
|
|
11646
|
-
|
|
11647
|
-
{
|
|
11648
|
-
role: "assistant",
|
|
11649
|
-
content: `Saved to memory: "${memory.key}" with ${memory.category} category, importance ${memory.importance}/10, scope: ${memory.scope}.`
|
|
11650
|
-
}
|
|
11651
|
-
]
|
|
11652
|
-
};
|
|
11795
|
+
async function uploadTrainingFile(filePath) {
|
|
11796
|
+
return withRetry(async () => {
|
|
11797
|
+
const client = getClient();
|
|
11798
|
+
const fileContent = readFileSync2(filePath);
|
|
11799
|
+
const blob2 = new Blob([fileContent], { type: "application/jsonl" });
|
|
11800
|
+
const file = new File([blob2], filePath.split("/").pop() ?? "training.jsonl", { type: "application/jsonl" });
|
|
11801
|
+
const response = await client.files.create({ file, purpose: "fine-tune" });
|
|
11802
|
+
return { fileId: response.id };
|
|
11803
|
+
});
|
|
11653
11804
|
}
|
|
11654
|
-
function
|
|
11655
|
-
|
|
11656
|
-
|
|
11657
|
-
|
|
11658
|
-
|
|
11659
|
-
|
|
11660
|
-
|
|
11661
|
-
|
|
11662
|
-
|
|
11663
|
-
${matched.map((m) => `- ${m.key}: ${m.value.slice(0, 120)}${m.value.length > 120 ? "..." : ""}`).join(`
|
|
11664
|
-
`)}` : `I don't have any ${category} memories stored yet.`
|
|
11665
|
-
}
|
|
11666
|
-
]
|
|
11667
|
-
};
|
|
11805
|
+
async function createFineTuneJob(fileId, baseModel, suffix) {
|
|
11806
|
+
return withRetry(async () => {
|
|
11807
|
+
const client = getClient();
|
|
11808
|
+
const params = { training_file: fileId, model: baseModel };
|
|
11809
|
+
if (suffix)
|
|
11810
|
+
params.suffix = suffix;
|
|
11811
|
+
const response = await client.fineTuning.jobs.create(params);
|
|
11812
|
+
return { jobId: response.id, status: response.status };
|
|
11813
|
+
});
|
|
11668
11814
|
}
|
|
11669
|
-
async function
|
|
11670
|
-
|
|
11671
|
-
|
|
11672
|
-
|
|
11673
|
-
let query = "SELECT * FROM memories WHERE status = 'active'";
|
|
11674
|
-
const params = [];
|
|
11675
|
-
if (options.since) {
|
|
11676
|
-
query += " AND created_at >= ?";
|
|
11677
|
-
params.push(options.since.toISOString());
|
|
11678
|
-
}
|
|
11679
|
-
query += " ORDER BY importance DESC, created_at DESC";
|
|
11680
|
-
if (options.limit) {
|
|
11681
|
-
query += " LIMIT ?";
|
|
11682
|
-
params.push(options.limit * 3);
|
|
11683
|
-
}
|
|
11684
|
-
const memories = db.query(query).all(...params);
|
|
11685
|
-
const examples = [];
|
|
11686
|
-
for (const memory of memories) {
|
|
11687
|
-
examples.push(memoryToRecallExample(memory));
|
|
11688
|
-
examples.push(memoryToSaveExample(memory));
|
|
11689
|
-
}
|
|
11690
|
-
const categories = [...new Set(memories.map((m) => m.category))];
|
|
11691
|
-
for (const category of categories) {
|
|
11692
|
-
examples.push(memoryToSearchExample(memories, category));
|
|
11693
|
-
}
|
|
11694
|
-
const finalExamples = options.limit ? examples.slice(0, options.limit) : examples;
|
|
11815
|
+
async function getFineTuneStatus(jobId) {
|
|
11816
|
+
return withRetry(async () => {
|
|
11817
|
+
const client = getClient();
|
|
11818
|
+
const response = await client.fineTuning.jobs.retrieve(jobId);
|
|
11695
11819
|
return {
|
|
11696
|
-
|
|
11697
|
-
|
|
11698
|
-
|
|
11820
|
+
jobId: response.id,
|
|
11821
|
+
status: response.status,
|
|
11822
|
+
fineTunedModel: response.fine_tuned_model ?? undefined,
|
|
11823
|
+
baseModel: response.model ?? undefined,
|
|
11824
|
+
error: response.error?.message ?? undefined
|
|
11699
11825
|
};
|
|
11700
|
-
}
|
|
11701
|
-
|
|
11702
|
-
|
|
11826
|
+
});
|
|
11827
|
+
}
|
|
11828
|
+
async function listFineTunedModels() {
|
|
11829
|
+
return withRetry(async () => {
|
|
11830
|
+
const client = getClient();
|
|
11831
|
+
const jobs = await client.fineTuning.jobs.list();
|
|
11832
|
+
return jobs.data.map((job) => ({
|
|
11833
|
+
id: job.id,
|
|
11834
|
+
model: job.fine_tuned_model ?? job.model,
|
|
11835
|
+
status: job.status,
|
|
11836
|
+
created: job.created_at
|
|
11837
|
+
}));
|
|
11838
|
+
});
|
|
11703
11839
|
}
|
|
11704
11840
|
|
|
11705
|
-
// src/lib/
|
|
11706
|
-
|
|
11707
|
-
|
|
11708
|
-
|
|
11709
|
-
|
|
11710
|
-
|
|
11711
|
-
|
|
11712
|
-
|
|
11713
|
-
|
|
11714
|
-
|
|
11715
|
-
|
|
11716
|
-
|
|
11717
|
-
const
|
|
11718
|
-
|
|
11719
|
-
|
|
11720
|
-
|
|
11721
|
-
|
|
11722
|
-
|
|
11723
|
-
|
|
11724
|
-
|
|
11725
|
-
|
|
11726
|
-
|
|
11727
|
-
|
|
11728
|
-
|
|
11729
|
-
|
|
11730
|
-
|
|
11731
|
-
|
|
11841
|
+
// src/lib/providers/thinker-labs.ts
|
|
11842
|
+
var DEFAULT_BASE_URL = "https://api.thinkerlabs.ai/v1";
|
|
11843
|
+
function getConfig() {
|
|
11844
|
+
const apiKey = getConfigValue("THINKER_LABS_API_KEY");
|
|
11845
|
+
const baseUrl = getConfigValue("THINKER_LABS_BASE_URL") ?? DEFAULT_BASE_URL;
|
|
11846
|
+
if (!apiKey)
|
|
11847
|
+
throw new Error("THINKER_LABS_API_KEY is not set. Run: brains config set THINKER_LABS_API_KEY <key>");
|
|
11848
|
+
return { apiKey, baseUrl };
|
|
11849
|
+
}
|
|
11850
|
+
async function request(method, path, body, file) {
|
|
11851
|
+
return withRetry(async () => {
|
|
11852
|
+
const { apiKey, baseUrl } = getConfig();
|
|
11853
|
+
const headers = { Authorization: `Bearer ${apiKey}` };
|
|
11854
|
+
let fetchBody;
|
|
11855
|
+
if (file) {
|
|
11856
|
+
const form = new FormData;
|
|
11857
|
+
form.append("file", new Blob([file.data], { type: "text/plain" }), file.name);
|
|
11858
|
+
if (body) {
|
|
11859
|
+
for (const [k, v] of Object.entries(body)) {
|
|
11860
|
+
form.append(k, v);
|
|
11861
|
+
}
|
|
11862
|
+
}
|
|
11863
|
+
fetchBody = form;
|
|
11864
|
+
} else if (body) {
|
|
11865
|
+
headers["Content-Type"] = "application/json";
|
|
11866
|
+
fetchBody = JSON.stringify(body);
|
|
11867
|
+
}
|
|
11868
|
+
const res = await fetch(`${baseUrl}${path}`, { method, headers, body: fetchBody });
|
|
11869
|
+
if (!res.ok) {
|
|
11870
|
+
const text2 = await res.text();
|
|
11871
|
+
throw new Error(`Thinker Labs API error ${res.status}: ${text2}`);
|
|
11872
|
+
}
|
|
11873
|
+
if (res.status === 204)
|
|
11874
|
+
return;
|
|
11875
|
+
return res.json();
|
|
11732
11876
|
});
|
|
11733
|
-
return { messages };
|
|
11734
11877
|
}
|
|
11735
|
-
async function
|
|
11736
|
-
const
|
|
11737
|
-
const
|
|
11738
|
-
|
|
11739
|
-
|
|
11740
|
-
|
|
11741
|
-
|
|
11742
|
-
|
|
11743
|
-
|
|
11744
|
-
|
|
11745
|
-
|
|
11746
|
-
|
|
11747
|
-
|
|
11748
|
-
|
|
11749
|
-
|
|
11750
|
-
|
|
11751
|
-
|
|
11752
|
-
|
|
11753
|
-
|
|
11754
|
-
|
|
11755
|
-
|
|
11756
|
-
|
|
11757
|
-
|
|
11758
|
-
|
|
11759
|
-
|
|
11760
|
-
|
|
11761
|
-
|
|
11762
|
-
|
|
11763
|
-
|
|
11764
|
-
|
|
11765
|
-
|
|
11766
|
-
|
|
11767
|
-
|
|
11768
|
-
|
|
11769
|
-
|
|
11770
|
-
count: finalExamples.length
|
|
11771
|
-
};
|
|
11772
|
-
} finally {
|
|
11773
|
-
db.close();
|
|
11774
|
-
}
|
|
11878
|
+
async function uploadTrainingData(filePath) {
|
|
11879
|
+
const fileContent = await Bun.file(filePath).text();
|
|
11880
|
+
const fileName = filePath.split("/").pop() ?? "training.jsonl";
|
|
11881
|
+
const result = await request("POST", "/datasets/upload", undefined, { name: fileName, data: fileContent });
|
|
11882
|
+
return { datasetId: result.id };
|
|
11883
|
+
}
|
|
11884
|
+
async function startFineTune(datasetId, baseModel, name) {
|
|
11885
|
+
const result = await request("POST", "/fine-tunes", {
|
|
11886
|
+
dataset_id: datasetId,
|
|
11887
|
+
base_model: baseModel,
|
|
11888
|
+
...name ? { name } : {}
|
|
11889
|
+
});
|
|
11890
|
+
return { jobId: result.id, status: result.status };
|
|
11891
|
+
}
|
|
11892
|
+
async function getStatus(jobId) {
|
|
11893
|
+
const result = await request("GET", `/fine-tunes/${jobId}`);
|
|
11894
|
+
return {
|
|
11895
|
+
jobId: result.id,
|
|
11896
|
+
status: result.status,
|
|
11897
|
+
modelId: result.model_id,
|
|
11898
|
+
error: result.error
|
|
11899
|
+
};
|
|
11900
|
+
}
|
|
11901
|
+
async function listModels() {
|
|
11902
|
+
const result = await request("GET", "/fine-tunes");
|
|
11903
|
+
return (result.data ?? []).map((m) => ({
|
|
11904
|
+
id: m.id,
|
|
11905
|
+
name: m.name,
|
|
11906
|
+
status: m.status,
|
|
11907
|
+
baseModel: m.base_model,
|
|
11908
|
+
createdAt: m.created_at
|
|
11909
|
+
}));
|
|
11910
|
+
}
|
|
11911
|
+
async function cancelJob(jobId) {
|
|
11912
|
+
await request("DELETE", `/fine-tunes/${jobId}`);
|
|
11775
11913
|
}
|
|
11776
11914
|
|
|
11777
|
-
|
|
11778
|
-
|
|
11779
|
-
|
|
11780
|
-
|
|
11781
|
-
|
|
11782
|
-
|
|
11783
|
-
|
|
11784
|
-
|
|
11785
|
-
return
|
|
11786
|
-
|
|
11787
|
-
|
|
11915
|
+
class ThinkerLabsProvider {
|
|
11916
|
+
uploadTrainingData = uploadTrainingData;
|
|
11917
|
+
startFineTune = startFineTune;
|
|
11918
|
+
getStatus = getStatus;
|
|
11919
|
+
listModels = listModels;
|
|
11920
|
+
cancelJob = cancelJob;
|
|
11921
|
+
async uploadTrainingFile(filePath) {
|
|
11922
|
+
const { datasetId } = await uploadTrainingData(filePath);
|
|
11923
|
+
return { fileId: datasetId };
|
|
11924
|
+
}
|
|
11925
|
+
async createFineTuneJob(fileId, baseModel, suffix) {
|
|
11926
|
+
return startFineTune(fileId, baseModel, suffix);
|
|
11927
|
+
}
|
|
11928
|
+
async getFineTuneStatus(jobId) {
|
|
11929
|
+
const result = await getStatus(jobId);
|
|
11930
|
+
return { jobId: result.jobId, status: result.status, fineTunedModel: result.modelId, error: result.error };
|
|
11931
|
+
}
|
|
11932
|
+
async listFineTunedModels() {
|
|
11933
|
+
const models = await listModels();
|
|
11934
|
+
return models.map((m) => ({ id: m.id, model: m.baseModel, status: m.status, created: m.createdAt }));
|
|
11935
|
+
}
|
|
11936
|
+
async cancelFineTuneJob(jobId) {
|
|
11937
|
+
return cancelJob(jobId);
|
|
11938
|
+
}
|
|
11788
11939
|
}
|
|
11789
|
-
|
|
11790
|
-
|
|
11791
|
-
|
|
11792
|
-
|
|
11793
|
-
if (
|
|
11794
|
-
|
|
11940
|
+
|
|
11941
|
+
// src/cli/ui.ts
|
|
11942
|
+
import chalk from "chalk";
|
|
11943
|
+
function printTable(headers, rows) {
|
|
11944
|
+
if (rows.length === 0) {
|
|
11945
|
+
console.log(chalk.dim(" (no records)"));
|
|
11946
|
+
return;
|
|
11795
11947
|
}
|
|
11796
|
-
const
|
|
11797
|
-
|
|
11798
|
-
|
|
11799
|
-
|
|
11800
|
-
|
|
11801
|
-
|
|
11802
|
-
|
|
11803
|
-
|
|
11804
|
-
|
|
11805
|
-
|
|
11806
|
-
|
|
11807
|
-
|
|
11808
|
-
if (options.since) {
|
|
11809
|
-
const fileStat = await stat(filePath).catch(() => null);
|
|
11810
|
-
if (fileStat && fileStat.mtime < options.since)
|
|
11811
|
-
continue;
|
|
11812
|
-
}
|
|
11813
|
-
const content = await readFile(filePath, "utf-8").catch(() => "");
|
|
11814
|
-
if (!content.trim())
|
|
11815
|
-
continue;
|
|
11816
|
-
const lines = content.trim().split(`
|
|
11817
|
-
`);
|
|
11818
|
-
const turns = [];
|
|
11819
|
-
for (const line of lines) {
|
|
11820
|
-
try {
|
|
11821
|
-
const entry = JSON.parse(line);
|
|
11822
|
-
if ((entry.type === "user" || entry.type === "human") && entry.message?.content) {
|
|
11823
|
-
const text2 = extractText(entry.message.content);
|
|
11824
|
-
if (text2.trim())
|
|
11825
|
-
turns.push({ role: "user", content: text2.trim() });
|
|
11826
|
-
} else if (entry.type === "assistant" && entry.message?.content) {
|
|
11827
|
-
const text2 = extractText(entry.message.content);
|
|
11828
|
-
if (text2.trim())
|
|
11829
|
-
turns.push({ role: "assistant", content: text2.trim() });
|
|
11830
|
-
}
|
|
11831
|
-
} catch {}
|
|
11832
|
-
}
|
|
11833
|
-
const windowSize = 6;
|
|
11834
|
-
for (let start = 0;start < turns.length - 1 && examples.length < limit2; start++) {
|
|
11835
|
-
const window2 = turns.slice(start, start + windowSize);
|
|
11836
|
-
if (!window2[0] || window2[0].role !== "user")
|
|
11837
|
-
continue;
|
|
11838
|
-
const lastAssistantIdx = window2.map((t) => t.role).lastIndexOf("assistant");
|
|
11839
|
-
if (lastAssistantIdx < 1)
|
|
11840
|
-
continue;
|
|
11841
|
-
const usedTurns = window2.slice(0, lastAssistantIdx + 1);
|
|
11842
|
-
examples.push({
|
|
11843
|
-
messages: [
|
|
11844
|
-
{ role: "system", content: SYSTEM_PROMPT4 },
|
|
11845
|
-
...usedTurns
|
|
11846
|
-
]
|
|
11847
|
-
});
|
|
11848
|
-
}
|
|
11849
|
-
}
|
|
11948
|
+
const colWidths = headers.map((h, i) => Math.max(h.length, ...rows.map((r) => (r[i] ?? "").length)));
|
|
11949
|
+
const separator = colWidths.map((w) => "\u2500".repeat(w + 2)).join("\u253C");
|
|
11950
|
+
const headerLine = headers.map((h, i) => ` ${chalk.bold(h.padEnd(colWidths[i] ?? 0))} `).join("\u2502");
|
|
11951
|
+
const topBorder = colWidths.map((w) => "\u2500".repeat(w + 2)).join("\u252C");
|
|
11952
|
+
const midBorder = separator;
|
|
11953
|
+
const bottomBorder = colWidths.map((w) => "\u2500".repeat(w + 2)).join("\u2534");
|
|
11954
|
+
console.log("\u250C" + topBorder + "\u2510");
|
|
11955
|
+
console.log("\u2502" + headerLine + "\u2502");
|
|
11956
|
+
console.log("\u251C" + midBorder + "\u2524");
|
|
11957
|
+
for (const row of rows) {
|
|
11958
|
+
const rowLine = row.map((cell, i) => ` ${(cell ?? "").padEnd(colWidths[i] ?? 0)} `).join("\u2502");
|
|
11959
|
+
console.log("\u2502" + rowLine + "\u2502");
|
|
11850
11960
|
}
|
|
11851
|
-
|
|
11961
|
+
console.log("\u2514" + bottomBorder + "\u2518");
|
|
11852
11962
|
}
|
|
11853
|
-
|
|
11854
|
-
|
|
11855
|
-
|
|
11856
|
-
|
|
11857
|
-
|
|
11858
|
-
|
|
11859
|
-
|
|
11860
|
-
|
|
11861
|
-
|
|
11862
|
-
|
|
11863
|
-
|
|
11864
|
-
|
|
11865
|
-
|
|
11866
|
-
|
|
11867
|
-
|
|
11868
|
-
|
|
11869
|
-
|
|
11870
|
-
|
|
11871
|
-
|
|
11963
|
+
var STATUS_COLORS = {
|
|
11964
|
+
succeeded: chalk.green,
|
|
11965
|
+
running: chalk.cyan,
|
|
11966
|
+
pending: chalk.yellow,
|
|
11967
|
+
failed: chalk.red,
|
|
11968
|
+
cancelled: chalk.dim,
|
|
11969
|
+
queued: chalk.yellow,
|
|
11970
|
+
validating_files: chalk.blue
|
|
11971
|
+
};
|
|
11972
|
+
function printStatus(status) {
|
|
11973
|
+
const colorFn = STATUS_COLORS[status] ?? chalk.white;
|
|
11974
|
+
return colorFn(`\u25CF ${status}`);
|
|
11975
|
+
}
|
|
11976
|
+
function printJson(obj) {
|
|
11977
|
+
console.log(JSON.stringify(obj, null, 2));
|
|
11978
|
+
}
|
|
11979
|
+
function printError(message) {
|
|
11980
|
+
console.error(chalk.red("\u2717 Error: ") + message);
|
|
11981
|
+
}
|
|
11982
|
+
function printSuccess(message) {
|
|
11983
|
+
console.log(chalk.green("\u2713 ") + message);
|
|
11984
|
+
}
|
|
11985
|
+
function printInfo(message) {
|
|
11986
|
+
console.log(chalk.dim(" " + message));
|
|
11872
11987
|
}
|
|
11873
11988
|
|
|
11874
11989
|
// src/cli/index.ts
|
|
11875
11990
|
var program2 = new Command;
|
|
11876
11991
|
program2.name("brains").description("Fine-tuned model tracker and trainer").version("0.0.1");
|
|
11877
11992
|
var modelsCmd = program2.command("models").description("Manage tracked fine-tuned models");
|
|
11878
|
-
modelsCmd.command("list").description("List all tracked fine-tuned models").action(async () => {
|
|
11993
|
+
modelsCmd.command("list").description("List all tracked fine-tuned models").option("--json", "Output as JSON").action(async (opts) => {
|
|
11879
11994
|
try {
|
|
11880
11995
|
const db = getDb();
|
|
11881
11996
|
const models = await db.select().from(fineTunedModels);
|
|
11997
|
+
if (opts.json) {
|
|
11998
|
+
printJson(models);
|
|
11999
|
+
return;
|
|
12000
|
+
}
|
|
11882
12001
|
if (models.length === 0) {
|
|
11883
12002
|
printInfo("No models tracked yet. Use 'brains finetune start' to train one.");
|
|
11884
12003
|
return;
|
|
@@ -11896,14 +12015,22 @@ modelsCmd.command("list").description("List all tracked fine-tuned models").acti
|
|
|
11896
12015
|
process.exit(1);
|
|
11897
12016
|
}
|
|
11898
12017
|
});
|
|
11899
|
-
modelsCmd.command("show <id>").description("Show details of a specific model").action(async (id) => {
|
|
12018
|
+
modelsCmd.command("show <id>").description("Show details of a specific model").option("--json", "Output as JSON").action(async (id, opts) => {
|
|
11900
12019
|
try {
|
|
11901
12020
|
const db = getDb();
|
|
11902
12021
|
const [model] = await db.select().from(fineTunedModels).where(eq(fineTunedModels.id, id));
|
|
11903
12022
|
if (!model) {
|
|
11904
|
-
|
|
12023
|
+
if (opts.json) {
|
|
12024
|
+
printJson({ error: `Model not found: ${id}` });
|
|
12025
|
+
} else {
|
|
12026
|
+
printError(`Model not found: ${id}`);
|
|
12027
|
+
}
|
|
11905
12028
|
process.exit(1);
|
|
11906
12029
|
}
|
|
12030
|
+
if (opts.json) {
|
|
12031
|
+
printJson(model);
|
|
12032
|
+
return;
|
|
12033
|
+
}
|
|
11907
12034
|
console.log();
|
|
11908
12035
|
const tagsList = model.tags ? JSON.parse(model.tags).join(", ") : "(none)";
|
|
11909
12036
|
console.log(` ID: ${model.id}`);
|
|
@@ -11990,29 +12117,89 @@ modelsCmd.command("collection <id> <collectionName>").description("Set the colle
|
|
|
11990
12117
|
process.exit(1);
|
|
11991
12118
|
}
|
|
11992
12119
|
});
|
|
12120
|
+
modelsCmd.command("import <job-id>").description("Import an externally created fine-tuned model into local tracking").option("--provider <provider>", "Provider (openai|thinker-labs)", "openai").option("--name <name>", "Display name for the model").action(async (jobId, opts) => {
|
|
12121
|
+
try {
|
|
12122
|
+
let result;
|
|
12123
|
+
if (opts.provider === "openai") {
|
|
12124
|
+
result = await getFineTuneStatus(jobId);
|
|
12125
|
+
} else {
|
|
12126
|
+
const tl = new ThinkerLabsProvider;
|
|
12127
|
+
result = await tl.getFineTuneStatus(jobId);
|
|
12128
|
+
}
|
|
12129
|
+
const db = getDb();
|
|
12130
|
+
const [existing] = await db.select().from(fineTunedModels).where(eq(fineTunedModels.fineTuneJobId, jobId));
|
|
12131
|
+
if (existing) {
|
|
12132
|
+
printInfo(`Model already tracked as: ${existing.id}`);
|
|
12133
|
+
return;
|
|
12134
|
+
}
|
|
12135
|
+
const modelId = randomUUID();
|
|
12136
|
+
const now = Date.now();
|
|
12137
|
+
const name = opts.name ?? result.fineTunedModel ?? `imported-${jobId}`;
|
|
12138
|
+
await db.insert(fineTunedModels).values({
|
|
12139
|
+
id: modelId,
|
|
12140
|
+
name,
|
|
12141
|
+
provider: opts.provider,
|
|
12142
|
+
baseModel: result.baseModel ?? "unknown",
|
|
12143
|
+
status: result.status,
|
|
12144
|
+
fineTuneJobId: jobId,
|
|
12145
|
+
createdAt: now,
|
|
12146
|
+
updatedAt: now
|
|
12147
|
+
});
|
|
12148
|
+
await db.insert(trainingJobs).values({
|
|
12149
|
+
id: randomUUID(),
|
|
12150
|
+
modelId,
|
|
12151
|
+
provider: opts.provider,
|
|
12152
|
+
status: result.status,
|
|
12153
|
+
startedAt: now
|
|
12154
|
+
});
|
|
12155
|
+
printSuccess(`Model imported successfully.`);
|
|
12156
|
+
console.log();
|
|
12157
|
+
console.log(` Local ID: ${modelId}`);
|
|
12158
|
+
console.log(` Job ID: ${jobId}`);
|
|
12159
|
+
console.log(` Name: ${name}`);
|
|
12160
|
+
console.log(` Status: ${printStatus(result.status)}`);
|
|
12161
|
+
if (result.fineTunedModel)
|
|
12162
|
+
console.log(` Model: ${result.fineTunedModel}`);
|
|
12163
|
+
console.log();
|
|
12164
|
+
} catch (err) {
|
|
12165
|
+
printError(err instanceof Error ? err.message : String(err));
|
|
12166
|
+
process.exit(1);
|
|
12167
|
+
}
|
|
12168
|
+
});
|
|
11993
12169
|
var finetuneCmd = program2.command("finetune").description("Manage fine-tuning jobs");
|
|
11994
|
-
finetuneCmd.command("start").description("Start a fine-tuning job").requiredOption("--provider <provider>", "Provider to use (openai|thinker-labs)").requiredOption("--base-model <model>", "Base model to fine-tune (e.g. gpt-4o-mini-2024-07-18)").
|
|
12170
|
+
finetuneCmd.command("start").description("Start a fine-tuning job").requiredOption("--provider <provider>", "Provider to use (openai|thinker-labs)").requiredOption("--base-model <model>", "Base model to fine-tune (e.g. gpt-4o-mini-2024-07-18)").option("--dataset <path>", "Path to the JSONL training dataset (auto-detects latest if omitted)").requiredOption("--name <name>", "Human-readable name for this fine-tuned model").action(async (opts) => {
|
|
11995
12171
|
try {
|
|
11996
12172
|
if (opts.provider !== "openai" && opts.provider !== "thinker-labs") {
|
|
11997
12173
|
printError(`Unknown provider: ${opts.provider}. Use 'openai' or 'thinker-labs'.`);
|
|
11998
12174
|
process.exit(1);
|
|
11999
12175
|
}
|
|
12000
|
-
|
|
12001
|
-
|
|
12176
|
+
let datasetPath = opts.dataset;
|
|
12177
|
+
if (!datasetPath) {
|
|
12178
|
+
const db2 = getDb();
|
|
12179
|
+
const [latest] = await db2.select().from(trainingDatasets).orderBy(desc(trainingDatasets.createdAt)).limit(1);
|
|
12180
|
+
if (!latest?.filePath) {
|
|
12181
|
+
printError("No datasets found. Run 'brains data gather' first.");
|
|
12182
|
+
process.exit(1);
|
|
12183
|
+
}
|
|
12184
|
+
datasetPath = latest.filePath;
|
|
12185
|
+
printInfo(`Using latest dataset: ${datasetPath} (${latest.exampleCount} examples)`);
|
|
12186
|
+
}
|
|
12187
|
+
if (!existsSync3(datasetPath)) {
|
|
12188
|
+
printError(`Dataset file not found: ${datasetPath}`);
|
|
12002
12189
|
process.exit(1);
|
|
12003
12190
|
}
|
|
12004
|
-
printInfo(`Uploading training file: ${
|
|
12191
|
+
printInfo(`Uploading training file: ${datasetPath} \u2026`);
|
|
12005
12192
|
let fileId;
|
|
12006
12193
|
let jobId;
|
|
12007
12194
|
let jobStatus;
|
|
12008
12195
|
if (opts.provider === "openai") {
|
|
12009
|
-
({ fileId } = await uploadTrainingFile(
|
|
12196
|
+
({ fileId } = await uploadTrainingFile(datasetPath));
|
|
12010
12197
|
printSuccess(`File uploaded. fileId = ${fileId}`);
|
|
12011
12198
|
printInfo(`Creating fine-tune job on OpenAI \u2026`);
|
|
12012
12199
|
({ jobId, status: jobStatus } = await createFineTuneJob(fileId, opts.baseModel, opts.name));
|
|
12013
12200
|
} else {
|
|
12014
12201
|
const tl = new ThinkerLabsProvider;
|
|
12015
|
-
({ fileId } = await tl.uploadTrainingFile(
|
|
12202
|
+
({ fileId } = await tl.uploadTrainingFile(datasetPath));
|
|
12016
12203
|
printSuccess(`File uploaded. fileId = ${fileId}`);
|
|
12017
12204
|
printInfo(`Creating fine-tune job on Thinker Labs \u2026`);
|
|
12018
12205
|
({ jobId, status: jobStatus } = await tl.createFineTuneJob(fileId, opts.baseModel, opts.name));
|
|
@@ -12050,7 +12237,7 @@ finetuneCmd.command("start").description("Start a fine-tuning job").requiredOpti
|
|
|
12050
12237
|
process.exit(1);
|
|
12051
12238
|
}
|
|
12052
12239
|
});
|
|
12053
|
-
finetuneCmd.command("status <job-id>").description("Get the status of a fine-tuning job").option("--provider <provider>", "Provider (openai|thinker-labs)", "openai").action(async (jobId, opts) => {
|
|
12240
|
+
finetuneCmd.command("status <job-id>").description("Get the status of a fine-tuning job").option("--provider <provider>", "Provider (openai|thinker-labs)", "openai").option("--json", "Output as JSON").action(async (jobId, opts) => {
|
|
12054
12241
|
try {
|
|
12055
12242
|
let result;
|
|
12056
12243
|
if (opts.provider === "openai") {
|
|
@@ -12059,16 +12246,20 @@ finetuneCmd.command("status <job-id>").description("Get the status of a fine-tun
|
|
|
12059
12246
|
const tl = new ThinkerLabsProvider;
|
|
12060
12247
|
result = await tl.getFineTuneStatus(jobId);
|
|
12061
12248
|
}
|
|
12062
|
-
|
|
12063
|
-
|
|
12064
|
-
|
|
12065
|
-
|
|
12066
|
-
console.log(`
|
|
12067
|
-
|
|
12068
|
-
|
|
12069
|
-
|
|
12249
|
+
if (opts.json) {
|
|
12250
|
+
printJson(result);
|
|
12251
|
+
} else {
|
|
12252
|
+
console.log();
|
|
12253
|
+
console.log(` Job ID: ${result.jobId}`);
|
|
12254
|
+
console.log(` Status: ${printStatus(result.status)}`);
|
|
12255
|
+
if (result.fineTunedModel) {
|
|
12256
|
+
console.log(` Fine-tuned model: ${result.fineTunedModel}`);
|
|
12257
|
+
}
|
|
12258
|
+
if (result.error) {
|
|
12259
|
+
console.log(` Error: ${result.error}`);
|
|
12260
|
+
}
|
|
12261
|
+
console.log();
|
|
12070
12262
|
}
|
|
12071
|
-
console.log();
|
|
12072
12263
|
const db = getDb();
|
|
12073
12264
|
const [model] = await db.select().from(fineTunedModels).where(eq(fineTunedModels.fineTuneJobId, jobId));
|
|
12074
12265
|
if (model) {
|
|
@@ -12080,7 +12271,64 @@ finetuneCmd.command("status <job-id>").description("Get the status of a fine-tun
|
|
|
12080
12271
|
process.exit(1);
|
|
12081
12272
|
}
|
|
12082
12273
|
});
|
|
12083
|
-
finetuneCmd.command("
|
|
12274
|
+
finetuneCmd.command("watch <job-id>").description("Poll a fine-tuning job until it completes or fails").option("--provider <provider>", "Provider (openai|thinker-labs)", "openai").option("--interval <seconds>", "Poll interval in seconds", "30").action(async (jobId, opts) => {
|
|
12275
|
+
const intervalMs = Math.max(5, parseInt(opts.interval, 10) || 30) * 1000;
|
|
12276
|
+
const terminalStates = new Set(["succeeded", "failed", "cancelled"]);
|
|
12277
|
+
printInfo(`Watching job ${jobId} (polling every ${intervalMs / 1000}s) \u2026`);
|
|
12278
|
+
console.log();
|
|
12279
|
+
const poll = async () => {
|
|
12280
|
+
try {
|
|
12281
|
+
let result;
|
|
12282
|
+
if (opts.provider === "openai") {
|
|
12283
|
+
result = await getFineTuneStatus(jobId);
|
|
12284
|
+
} else {
|
|
12285
|
+
const tl = new ThinkerLabsProvider;
|
|
12286
|
+
result = await tl.getFineTuneStatus(jobId);
|
|
12287
|
+
}
|
|
12288
|
+
const ts = new Date().toISOString().replace("T", " ").slice(0, 19);
|
|
12289
|
+
process.stdout.write(` [${ts}] ${printStatus(result.status)}`);
|
|
12290
|
+
if (result.fineTunedModel)
|
|
12291
|
+
process.stdout.write(` model: ${result.fineTunedModel}`);
|
|
12292
|
+
process.stdout.write(`
|
|
12293
|
+
`);
|
|
12294
|
+
const db = getDb();
|
|
12295
|
+
const [model] = await db.select().from(fineTunedModels).where(eq(fineTunedModels.fineTuneJobId, jobId));
|
|
12296
|
+
if (model) {
|
|
12297
|
+
await db.update(fineTunedModels).set({ status: result.status, updatedAt: Date.now() }).where(eq(fineTunedModels.fineTuneJobId, jobId));
|
|
12298
|
+
}
|
|
12299
|
+
if (terminalStates.has(result.status)) {
|
|
12300
|
+
console.log();
|
|
12301
|
+
if (result.status === "succeeded") {
|
|
12302
|
+
printSuccess(`Job completed successfully.`);
|
|
12303
|
+
if (result.fineTunedModel)
|
|
12304
|
+
printSuccess(`Fine-tuned model: ${result.fineTunedModel}`);
|
|
12305
|
+
} else if (result.status === "failed") {
|
|
12306
|
+
printError(`Job failed.${result.error ? " Error: " + result.error : ""}`);
|
|
12307
|
+
} else {
|
|
12308
|
+
printInfo(`Job ${result.status}.`);
|
|
12309
|
+
}
|
|
12310
|
+
return true;
|
|
12311
|
+
}
|
|
12312
|
+
return false;
|
|
12313
|
+
} catch (err) {
|
|
12314
|
+
printError(err instanceof Error ? err.message : String(err));
|
|
12315
|
+
return false;
|
|
12316
|
+
}
|
|
12317
|
+
};
|
|
12318
|
+
const done = await poll();
|
|
12319
|
+
if (!done) {
|
|
12320
|
+
await new Promise((resolve2) => {
|
|
12321
|
+
const timer = setInterval(async () => {
|
|
12322
|
+
const finished = await poll();
|
|
12323
|
+
if (finished) {
|
|
12324
|
+
clearInterval(timer);
|
|
12325
|
+
resolve2();
|
|
12326
|
+
}
|
|
12327
|
+
}, intervalMs);
|
|
12328
|
+
});
|
|
12329
|
+
}
|
|
12330
|
+
});
|
|
12331
|
+
finetuneCmd.command("list").description("List all fine-tuning jobs").option("--provider <provider>", "Provider to query (openai|thinker-labs)", "openai").option("--json", "Output as JSON").action(async (opts) => {
|
|
12084
12332
|
try {
|
|
12085
12333
|
let jobs;
|
|
12086
12334
|
if (opts.provider === "openai") {
|
|
@@ -12089,6 +12337,10 @@ finetuneCmd.command("list").description("List all fine-tuning jobs").option("--p
|
|
|
12089
12337
|
const tl = new ThinkerLabsProvider;
|
|
12090
12338
|
jobs = await tl.listFineTunedModels();
|
|
12091
12339
|
}
|
|
12340
|
+
if (opts.json) {
|
|
12341
|
+
printJson(jobs);
|
|
12342
|
+
return;
|
|
12343
|
+
}
|
|
12092
12344
|
if (jobs.length === 0) {
|
|
12093
12345
|
printInfo("No fine-tuning jobs found.");
|
|
12094
12346
|
return;
|
|
@@ -12104,7 +12356,7 @@ finetuneCmd.command("list").description("List all fine-tuning jobs").option("--p
|
|
|
12104
12356
|
process.exit(1);
|
|
12105
12357
|
}
|
|
12106
12358
|
});
|
|
12107
|
-
var DEFAULT_DATASETS_DIR =
|
|
12359
|
+
var DEFAULT_DATASETS_DIR = join7(homedir7(), ".brains", "datasets");
|
|
12108
12360
|
var dataCmd = program2.command("data").description("Manage training datasets");
|
|
12109
12361
|
dataCmd.command("gather").description("Gather training data from agent memory sources").option("--source <source>", "Data source: todos|mementos|conversations|sessions|all", "all").option("--output <dir>", "Output directory", DEFAULT_DATASETS_DIR).option("--limit <n>", "Maximum number of examples to gather", "500").action(async (opts) => {
|
|
12110
12362
|
const validSources = ["todos", "mementos", "conversations", "sessions", "all"];
|
|
@@ -12118,44 +12370,59 @@ dataCmd.command("gather").description("Gather training data from agent memory so
|
|
|
12118
12370
|
process.exit(1);
|
|
12119
12371
|
}
|
|
12120
12372
|
try {
|
|
12121
|
-
|
|
12373
|
+
mkdirSync3(opts.output, { recursive: true });
|
|
12122
12374
|
const sources = opts.source === "all" ? ["todos", "mementos", "conversations", "sessions"] : [opts.source];
|
|
12123
12375
|
const now = Date.now();
|
|
12124
12376
|
const db = getDb();
|
|
12125
|
-
const
|
|
12377
|
+
const gathererMap = {
|
|
12378
|
+
todos: (o) => Promise.resolve().then(() => (init_todos(), exports_todos)).then((m) => m.gatherFromTodos(o)),
|
|
12379
|
+
mementos: (o) => Promise.resolve().then(() => (init_mementos(), exports_mementos)).then((m) => m.gatherFromMementos(o)),
|
|
12380
|
+
conversations: (o) => Promise.resolve().then(() => (init_conversations(), exports_conversations)).then((m) => m.gatherFromConversations(o)),
|
|
12381
|
+
sessions: (o) => Promise.resolve().then(() => (init_sessions(), exports_sessions2)).then((m) => m.gatherFromSessions(o))
|
|
12382
|
+
};
|
|
12126
12383
|
let totalExamples = 0;
|
|
12127
|
-
|
|
12128
|
-
|
|
12129
|
-
printInfo(`
|
|
12130
|
-
|
|
12131
|
-
|
|
12132
|
-
|
|
12133
|
-
|
|
12134
|
-
|
|
12135
|
-
|
|
12136
|
-
|
|
12137
|
-
|
|
12384
|
+
let successfulSources = 0;
|
|
12385
|
+
for (const source of sources) {
|
|
12386
|
+
printInfo(`Gathering from ${source} \u2026`);
|
|
12387
|
+
try {
|
|
12388
|
+
const gatherer = gathererMap[source];
|
|
12389
|
+
if (!gatherer) {
|
|
12390
|
+
printError(` Unknown source: ${source}`);
|
|
12391
|
+
continue;
|
|
12392
|
+
}
|
|
12393
|
+
const { examples, count } = await gatherer({ limit: limit2 });
|
|
12394
|
+
if (count === 0) {
|
|
12395
|
+
printInfo(` No examples found in ${source}.`);
|
|
12396
|
+
continue;
|
|
12397
|
+
}
|
|
12398
|
+
const fileName = `${source}-${now}.jsonl`;
|
|
12399
|
+
const filePath = join7(opts.output, fileName);
|
|
12400
|
+
writeFileSync2(filePath, examples.map((e) => JSON.stringify(e)).join(`
|
|
12138
12401
|
`) + `
|
|
12139
12402
|
`, "utf8");
|
|
12140
|
-
|
|
12141
|
-
|
|
12142
|
-
|
|
12143
|
-
|
|
12144
|
-
|
|
12145
|
-
|
|
12146
|
-
|
|
12147
|
-
|
|
12148
|
-
|
|
12403
|
+
await db.insert(trainingDatasets).values({
|
|
12404
|
+
id: randomUUID(),
|
|
12405
|
+
source,
|
|
12406
|
+
filePath,
|
|
12407
|
+
exampleCount: count,
|
|
12408
|
+
createdAt: now
|
|
12409
|
+
});
|
|
12410
|
+
printSuccess(` \u2713 ${count} examples \u2192 ${filePath}`);
|
|
12411
|
+
totalExamples += count;
|
|
12412
|
+
successfulSources++;
|
|
12413
|
+
} catch (sourceErr) {
|
|
12414
|
+
printError(` \u2717 ${source}: ${sourceErr instanceof Error ? sourceErr.message : String(sourceErr)}`);
|
|
12415
|
+
}
|
|
12149
12416
|
}
|
|
12150
12417
|
console.log();
|
|
12151
|
-
printSuccess(`Total examples
|
|
12418
|
+
printSuccess(`Total: ${totalExamples} examples from ${successfulSources} source(s)`);
|
|
12152
12419
|
} catch (err) {
|
|
12153
12420
|
printError(err instanceof Error ? err.message : String(err));
|
|
12154
12421
|
process.exit(1);
|
|
12155
12422
|
}
|
|
12156
12423
|
});
|
|
12157
12424
|
dataCmd.command("preview <file>").description("Preview a JSONL training file").option("-n, --count <n>", "Number of examples to show", "5").action((file, opts) => {
|
|
12158
|
-
if (!
|
|
12425
|
+
if (!existsSync3(file)) {
|
|
12159
12426
|
printError(`File not found: ${file}`);
|
|
12160
12427
|
process.exit(1);
|
|
12161
12428
|
}
|
|
@@ -12165,7 +12432,7 @@ dataCmd.command("preview <file>").description("Preview a JSONL training file").o
|
|
|
12165
12432
|
process.exit(1);
|
|
12166
12433
|
}
|
|
12167
12434
|
try {
|
|
12168
|
-
const content =
|
|
12435
|
+
const content = readFileSync3(file, "utf8");
|
|
12169
12436
|
const lines = content.trim().split(`
|
|
12170
12437
|
`).filter(Boolean);
|
|
12171
12438
|
const total = lines.length;
|
|
@@ -12190,10 +12457,63 @@ dataCmd.command("preview <file>").description("Preview a JSONL training file").o
|
|
|
12190
12457
|
process.exit(1);
|
|
12191
12458
|
}
|
|
12192
12459
|
});
|
|
12193
|
-
dataCmd.command("
|
|
12460
|
+
dataCmd.command("merge <files...>").description("Merge multiple JSONL datasets into one").option("--output <path>", "Output file path", join7(DEFAULT_DATASETS_DIR, `merged-${Date.now()}.jsonl`)).option("--no-dedupe", "Skip deduplication").action(async (files, opts) => {
|
|
12461
|
+
try {
|
|
12462
|
+
for (const f of files) {
|
|
12463
|
+
if (!existsSync3(f)) {
|
|
12464
|
+
printError(`File not found: ${f}`);
|
|
12465
|
+
process.exit(1);
|
|
12466
|
+
}
|
|
12467
|
+
}
|
|
12468
|
+
mkdirSync3(join7(opts.output, "..").replace(/\/\.\.$/, "") || DEFAULT_DATASETS_DIR, { recursive: true });
|
|
12469
|
+
const allExamples = [];
|
|
12470
|
+
for (const f of files) {
|
|
12471
|
+
const lines = readFileSync3(f, "utf8").split(`
|
|
12472
|
+
`).map((l) => l.trim()).filter(Boolean);
|
|
12473
|
+
allExamples.push(...lines);
|
|
12474
|
+
printInfo(` Read ${lines.length} examples from ${f}`);
|
|
12475
|
+
}
|
|
12476
|
+
let finalLines = allExamples;
|
|
12477
|
+
let dupeCount = 0;
|
|
12478
|
+
if (opts.dedupe) {
|
|
12479
|
+
const seen = new Set;
|
|
12480
|
+
finalLines = allExamples.filter((line) => {
|
|
12481
|
+
if (seen.has(line)) {
|
|
12482
|
+
dupeCount++;
|
|
12483
|
+
return false;
|
|
12484
|
+
}
|
|
12485
|
+
seen.add(line);
|
|
12486
|
+
return true;
|
|
12487
|
+
});
|
|
12488
|
+
}
|
|
12489
|
+
writeFileSync2(opts.output, finalLines.join(`
|
|
12490
|
+
`) + `
|
|
12491
|
+
`, "utf8");
|
|
12492
|
+
const db = getDb();
|
|
12493
|
+
await db.insert(trainingDatasets).values({
|
|
12494
|
+
id: randomUUID(),
|
|
12495
|
+
source: "mixed",
|
|
12496
|
+
filePath: opts.output,
|
|
12497
|
+
exampleCount: finalLines.length,
|
|
12498
|
+
createdAt: Date.now()
|
|
12499
|
+
});
|
|
12500
|
+
console.log();
|
|
12501
|
+
printSuccess(`Merged ${files.length} files \u2014 ${finalLines.length} examples \u2192 ${opts.output}`);
|
|
12502
|
+
if (opts.dedupe && dupeCount > 0)
|
|
12503
|
+
printInfo(` Removed ${dupeCount} duplicate(s)`);
|
|
12504
|
+
} catch (err) {
|
|
12505
|
+
printError(err instanceof Error ? err.message : String(err));
|
|
12506
|
+
process.exit(1);
|
|
12507
|
+
}
|
|
12508
|
+
});
|
|
12509
|
+
dataCmd.command("list").description("List all gathered datasets").option("--json", "Output as JSON").action(async (opts) => {
|
|
12194
12510
|
try {
|
|
12195
12511
|
const db = getDb();
|
|
12196
12512
|
const datasets = await db.select().from(trainingDatasets);
|
|
12513
|
+
if (opts.json) {
|
|
12514
|
+
printJson(datasets);
|
|
12515
|
+
return;
|
|
12516
|
+
}
|
|
12197
12517
|
if (datasets.length === 0) {
|
|
12198
12518
|
printInfo("No datasets found. Use 'brains data gather' to create one.");
|
|
12199
12519
|
return;
|
|
@@ -12211,10 +12531,10 @@ dataCmd.command("list").description("List all gathered datasets").action(async (
|
|
|
12211
12531
|
}
|
|
12212
12532
|
});
|
|
12213
12533
|
var collectionsCmd = program2.command("collections").description("Manage model collections");
|
|
12214
|
-
collectionsCmd.action(async () => {
|
|
12215
|
-
await listCollections();
|
|
12534
|
+
collectionsCmd.option("--json", "Output as JSON").action(async (opts) => {
|
|
12535
|
+
await listCollections(opts.json);
|
|
12216
12536
|
});
|
|
12217
|
-
async function listCollections() {
|
|
12537
|
+
async function listCollections(json = false) {
|
|
12218
12538
|
try {
|
|
12219
12539
|
const db = getDb();
|
|
12220
12540
|
const rows = await db.select({
|
|
@@ -12222,6 +12542,10 @@ async function listCollections() {
|
|
|
12222
12542
|
count: sql`count(*)`.as("count"),
|
|
12223
12543
|
names: sql`group_concat(coalesce(${fineTunedModels.displayName}, ${fineTunedModels.name}), ', ')`.as("names")
|
|
12224
12544
|
}).from(fineTunedModels).groupBy(fineTunedModels.collection);
|
|
12545
|
+
if (json) {
|
|
12546
|
+
printJson(rows);
|
|
12547
|
+
return;
|
|
12548
|
+
}
|
|
12225
12549
|
if (rows.length === 0) {
|
|
12226
12550
|
printInfo("No collections found. Set a collection with 'brains models set-collection'.");
|
|
12227
12551
|
return;
|
|
@@ -12236,8 +12560,8 @@ async function listCollections() {
|
|
|
12236
12560
|
process.exit(1);
|
|
12237
12561
|
}
|
|
12238
12562
|
}
|
|
12239
|
-
collectionsCmd.command("list").description("List all collections with model counts").action(async () => {
|
|
12240
|
-
await listCollections();
|
|
12563
|
+
collectionsCmd.command("list").description("List all collections with model counts").option("--json", "Output as JSON").action(async (opts) => {
|
|
12564
|
+
await listCollections(opts.json);
|
|
12241
12565
|
});
|
|
12242
12566
|
collectionsCmd.command("show <name>").description("List all models in a collection").action(async (name) => {
|
|
12243
12567
|
try {
|
|
@@ -12296,4 +12620,43 @@ program2.command("remove <id>").alias("rm").alias("uninstall").description("Remo
|
|
|
12296
12620
|
process.exit(1);
|
|
12297
12621
|
}
|
|
12298
12622
|
});
|
|
12623
|
+
var configCmd = program2.command("config").description("Manage API keys and settings");
|
|
12624
|
+
configCmd.command("list").description("Show all config keys and their sources").action(() => {
|
|
12625
|
+
const entries = listConfig();
|
|
12626
|
+
console.log();
|
|
12627
|
+
for (const { key, value, source } of entries) {
|
|
12628
|
+
const display = source === "unset" ? "(unset)" : value.length > 8 ? value.slice(0, 4) + "****" + value.slice(-4) : "****";
|
|
12629
|
+
const src = source === "env" ? " [env]" : source === "file" ? " [file]" : "";
|
|
12630
|
+
console.log(` ${key.padEnd(28)} ${display}${src}`);
|
|
12631
|
+
}
|
|
12632
|
+
console.log();
|
|
12633
|
+
});
|
|
12634
|
+
configCmd.command("get <key>").description("Get a config value").action((key) => {
|
|
12635
|
+
if (!CONFIG_KEYS.includes(key)) {
|
|
12636
|
+
printError(`Unknown key: ${key}. Valid keys: ${CONFIG_KEYS.join(", ")}`);
|
|
12637
|
+
process.exit(1);
|
|
12638
|
+
}
|
|
12639
|
+
const value = getConfigValue(key);
|
|
12640
|
+
if (!value) {
|
|
12641
|
+
printInfo(`${key} is not set.`);
|
|
12642
|
+
} else {
|
|
12643
|
+
console.log(value);
|
|
12644
|
+
}
|
|
12645
|
+
});
|
|
12646
|
+
configCmd.command("set <key> <value>").description("Set a config value (stored in ~/.brains/config.json)").action((key, value) => {
|
|
12647
|
+
if (!CONFIG_KEYS.includes(key)) {
|
|
12648
|
+
printError(`Unknown key: ${key}. Valid keys: ${CONFIG_KEYS.join(", ")}`);
|
|
12649
|
+
process.exit(1);
|
|
12650
|
+
}
|
|
12651
|
+
setConfigValue(key, value);
|
|
12652
|
+
printSuccess(`${key} saved to ~/.brains/config.json`);
|
|
12653
|
+
});
|
|
12654
|
+
configCmd.command("unset <key>").description("Remove a config value from the config file").action((key) => {
|
|
12655
|
+
if (!CONFIG_KEYS.includes(key)) {
|
|
12656
|
+
printError(`Unknown key: ${key}. Valid keys: ${CONFIG_KEYS.join(", ")}`);
|
|
12657
|
+
process.exit(1);
|
|
12658
|
+
}
|
|
12659
|
+
deleteConfigValue(key);
|
|
12660
|
+
printSuccess(`${key} removed from config.`);
|
|
12661
|
+
});
|
|
12299
12662
|
program2.parse();
|