@vellumai/assistant 0.4.11 → 0.4.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ARCHITECTURE.md +401 -385
- package/package.json +1 -1
- package/src/__tests__/guardian-verify-setup-skill-regression.test.ts +75 -61
- package/src/__tests__/registry.test.ts +235 -187
- package/src/__tests__/secure-keys.test.ts +27 -0
- package/src/__tests__/session-agent-loop.test.ts +521 -256
- package/src/__tests__/session-surfaces-task-progress.test.ts +1 -0
- package/src/__tests__/session-tool-setup-app-refresh.test.ts +1 -0
- package/src/__tests__/session-tool-setup-memory-scope.test.ts +1 -0
- package/src/__tests__/session-tool-setup-side-effect-flag.test.ts +1 -0
- package/src/__tests__/skills.test.ts +334 -276
- package/src/__tests__/slack-skill.test.ts +124 -0
- package/src/__tests__/starter-task-flow.test.ts +7 -17
- package/src/agent/loop.ts +10 -3
- package/src/config/bundled-skills/chatgpt-import/tools/chatgpt-import.ts +449 -0
- package/src/config/bundled-skills/doordash/SKILL.md +171 -0
- package/src/config/bundled-skills/doordash/__tests__/doordash-client.test.ts +203 -0
- package/src/config/bundled-skills/doordash/__tests__/doordash-session.test.ts +164 -0
- package/src/config/bundled-skills/doordash/doordash-cli.ts +1193 -0
- package/src/config/bundled-skills/doordash/doordash-entry.ts +22 -0
- package/src/config/bundled-skills/doordash/lib/cart-queries.ts +787 -0
- package/src/config/bundled-skills/doordash/lib/client.ts +1071 -0
- package/src/config/bundled-skills/doordash/lib/order-queries.ts +85 -0
- package/src/config/bundled-skills/doordash/lib/queries.ts +28 -0
- package/src/config/bundled-skills/doordash/lib/query-extractor.ts +94 -0
- package/src/config/bundled-skills/doordash/lib/search-queries.ts +203 -0
- package/src/config/bundled-skills/doordash/lib/session.ts +93 -0
- package/src/config/bundled-skills/doordash/lib/shared/errors.ts +61 -0
- package/src/config/bundled-skills/doordash/lib/shared/ipc.ts +32 -0
- package/src/config/bundled-skills/doordash/lib/shared/network-recorder.ts +380 -0
- package/src/config/bundled-skills/doordash/lib/shared/platform.ts +35 -0
- package/src/config/bundled-skills/doordash/lib/shared/recording-store.ts +43 -0
- package/src/config/bundled-skills/doordash/lib/shared/recording-types.ts +49 -0
- package/src/config/bundled-skills/doordash/lib/shared/truncate.ts +6 -0
- package/src/config/bundled-skills/doordash/lib/store-queries.ts +246 -0
- package/src/config/bundled-skills/doordash/lib/types.ts +367 -0
- package/src/config/bundled-skills/google-calendar/SKILL.md +4 -5
- package/src/config/bundled-skills/google-oauth-setup/SKILL.md +41 -41
- package/src/config/bundled-skills/messaging/SKILL.md +59 -42
- package/src/config/bundled-skills/messaging/TOOLS.json +14 -92
- package/src/config/bundled-skills/messaging/tools/gmail-archive-by-query.ts +5 -1
- package/src/config/bundled-skills/messaging/tools/gmail-batch-archive.ts +11 -2
- package/src/config/bundled-skills/messaging/tools/gmail-outreach-scan.ts +8 -1
- package/src/config/bundled-skills/messaging/tools/gmail-sender-digest.ts +12 -4
- package/src/config/bundled-skills/messaging/tools/gmail-unsubscribe.ts +5 -1
- package/src/config/bundled-skills/messaging/tools/messaging-archive-by-sender.ts +5 -1
- package/src/config/bundled-skills/messaging/tools/messaging-sender-digest.ts +5 -2
- package/src/config/bundled-skills/notion/SKILL.md +240 -0
- package/src/config/bundled-skills/notion-oauth-setup/SKILL.md +127 -0
- package/src/config/bundled-skills/oauth-setup/SKILL.md +144 -0
- package/src/config/bundled-skills/phone-calls/SKILL.md +76 -45
- package/src/config/bundled-skills/skills-catalog/SKILL.md +32 -29
- package/src/config/bundled-skills/slack/SKILL.md +49 -0
- package/src/config/bundled-skills/slack/TOOLS.json +167 -0
- package/src/config/bundled-skills/slack/tools/shared.ts +23 -0
- package/src/config/bundled-skills/{messaging → slack}/tools/slack-add-reaction.ts +2 -5
- package/src/config/bundled-skills/slack/tools/slack-channel-details.ts +33 -0
- package/src/config/bundled-skills/slack/tools/slack-configure-channels.ts +75 -0
- package/src/config/bundled-skills/{messaging → slack}/tools/slack-delete-message.ts +2 -5
- package/src/config/bundled-skills/{messaging → slack}/tools/slack-leave-channel.ts +2 -5
- package/src/config/bundled-skills/slack/tools/slack-scan-digest.ts +193 -0
- package/src/config/{vellum-skills → bundled-skills}/sms-setup/SKILL.md +29 -22
- package/src/config/{vellum-skills → bundled-skills}/telegram-setup/SKILL.md +17 -14
- package/src/config/{vellum-skills → bundled-skills}/twilio-setup/SKILL.md +20 -5
- package/src/config/bundled-tool-registry.ts +292 -267
- package/src/config/schema.ts +1 -1
- package/src/daemon/handlers/skills.ts +334 -234
- package/src/daemon/ipc-contract/messages.ts +2 -0
- package/src/daemon/ipc-contract/surfaces.ts +2 -0
- package/src/daemon/lifecycle.ts +358 -221
- package/src/daemon/response-tier.ts +2 -0
- package/src/daemon/server.ts +453 -193
- package/src/daemon/session-agent-loop-handlers.ts +43 -2
- package/src/daemon/session-agent-loop.ts +3 -0
- package/src/daemon/session-lifecycle.ts +3 -0
- package/src/daemon/session-process.ts +1 -0
- package/src/daemon/session-surfaces.ts +22 -20
- package/src/daemon/session-tool-setup.ts +1 -0
- package/src/daemon/session.ts +5 -2
- package/src/messaging/outreach-classifier.ts +12 -5
- package/src/messaging/provider-types.ts +5 -0
- package/src/messaging/provider.ts +1 -1
- package/src/messaging/providers/gmail/adapter.ts +11 -5
- package/src/messaging/providers/gmail/client.ts +2 -0
- package/src/messaging/providers/slack/adapter.ts +1 -0
- package/src/messaging/providers/slack/client.ts +8 -0
- package/src/messaging/providers/slack/types.ts +5 -0
- package/src/runtime/http-errors.ts +33 -20
- package/src/runtime/http-server.ts +706 -291
- package/src/runtime/http-types.ts +26 -16
- package/src/runtime/routes/secret-routes.ts +57 -2
- package/src/runtime/routes/surface-action-routes.ts +66 -0
- package/src/runtime/routes/trust-rules-routes.ts +140 -0
- package/src/security/keychain-to-encrypted-migration.ts +59 -0
- package/src/security/secure-keys.ts +17 -0
- package/src/skills/frontmatter.ts +9 -7
- package/src/tools/apps/executors.ts +2 -1
- package/src/tools/tool-manifest.ts +44 -42
- package/src/tools/types.ts +9 -0
- package/src/__tests__/skill-mirror-parity.test.ts +0 -176
- package/src/config/vellum-skills/catalog.json +0 -63
- package/src/config/vellum-skills/chatgpt-import/tools/chatgpt-import.ts +0 -295
- package/src/skills/vellum-catalog-remote.ts +0 -166
- package/src/tools/skills/vellum-catalog.ts +0 -168
- /package/src/config/{vellum-skills → bundled-skills}/chatgpt-import/SKILL.md +0 -0
- /package/src/config/{vellum-skills → bundled-skills}/chatgpt-import/TOOLS.json +0 -0
- /package/src/config/{vellum-skills → bundled-skills}/deploy-fullstack-vercel/SKILL.md +0 -0
- /package/src/config/{vellum-skills → bundled-skills}/document-writer/SKILL.md +0 -0
- /package/src/config/{vellum-skills → bundled-skills}/guardian-verify-setup/SKILL.md +0 -0
- /package/src/config/{vellum-skills → bundled-skills}/slack-oauth-setup/SKILL.md +0 -0
- /package/src/config/{vellum-skills → bundled-skills}/trusted-contacts/SKILL.md +0 -0
|
@@ -1,176 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Drift guard: asserts parity between mirrored first-party skill directories.
|
|
3
|
-
*
|
|
4
|
-
* Skills that exist in both `assistant/src/config/vellum-skills/` (embedded in
|
|
5
|
-
* the runtime) and `skills/` (top-level repo, fetched from GitHub at runtime)
|
|
6
|
-
* must have identical SKILL.md content. The catalog.json entries for shared
|
|
7
|
-
* skills must also match.
|
|
8
|
-
*
|
|
9
|
-
* This test prevents silent drift between the two copies. When a skill is
|
|
10
|
-
* updated in one location but not the other, this test fails with a clear
|
|
11
|
-
* message indicating which skill and file diverged.
|
|
12
|
-
*/
|
|
13
|
-
|
|
14
|
-
import { existsSync, readdirSync, readFileSync, statSync } from 'node:fs';
|
|
15
|
-
import { join, resolve } from 'node:path';
|
|
16
|
-
|
|
17
|
-
import { describe, expect, test } from 'bun:test';
|
|
18
|
-
|
|
19
|
-
// ---------------------------------------------------------------------------
|
|
20
|
-
// Paths
|
|
21
|
-
// ---------------------------------------------------------------------------
|
|
22
|
-
|
|
23
|
-
const ASSISTANT_DIR = resolve(__dirname, '..', '..');
|
|
24
|
-
const REPO_ROOT = resolve(ASSISTANT_DIR, '..');
|
|
25
|
-
|
|
26
|
-
const EMBEDDED_SKILLS_DIR = join(ASSISTANT_DIR, 'src', 'config', 'vellum-skills');
|
|
27
|
-
const TOPLEVEL_SKILLS_DIR = join(REPO_ROOT, 'skills');
|
|
28
|
-
|
|
29
|
-
const EMBEDDED_CATALOG = join(EMBEDDED_SKILLS_DIR, 'catalog.json');
|
|
30
|
-
const TOPLEVEL_CATALOG = join(TOPLEVEL_SKILLS_DIR, 'catalog.json');
|
|
31
|
-
|
|
32
|
-
// ---------------------------------------------------------------------------
|
|
33
|
-
// Skills that exist only in the top-level `skills/` directory and have no
|
|
34
|
-
// embedded source in `assistant/src/config/vellum-skills/`. These are
|
|
35
|
-
// community/third-party skills maintained outside the assistant runtime.
|
|
36
|
-
// ---------------------------------------------------------------------------
|
|
37
|
-
|
|
38
|
-
const TOPLEVEL_ONLY_SKILLS = new Set([
|
|
39
|
-
'doordash',
|
|
40
|
-
'google-oauth-setup',
|
|
41
|
-
'notion',
|
|
42
|
-
'notion-oauth-setup',
|
|
43
|
-
'oauth-setup',
|
|
44
|
-
]);
|
|
45
|
-
|
|
46
|
-
// ---------------------------------------------------------------------------
|
|
47
|
-
// Helpers
|
|
48
|
-
// ---------------------------------------------------------------------------
|
|
49
|
-
|
|
50
|
-
interface CatalogEntry {
|
|
51
|
-
id: string;
|
|
52
|
-
name: string;
|
|
53
|
-
description: string;
|
|
54
|
-
emoji: string;
|
|
55
|
-
includes?: string[];
|
|
56
|
-
}
|
|
57
|
-
|
|
58
|
-
interface Catalog {
|
|
59
|
-
description: string;
|
|
60
|
-
version: number;
|
|
61
|
-
skills: CatalogEntry[];
|
|
62
|
-
}
|
|
63
|
-
|
|
64
|
-
/** List subdirectories (skill directories) in a given parent. */
|
|
65
|
-
function listSkillDirs(parent: string): string[] {
|
|
66
|
-
if (!existsSync(parent)) return [];
|
|
67
|
-
return readdirSync(parent).filter((entry) => {
|
|
68
|
-
const full = join(parent, entry);
|
|
69
|
-
return statSync(full).isDirectory();
|
|
70
|
-
});
|
|
71
|
-
}
|
|
72
|
-
|
|
73
|
-
/** Find skill IDs that exist as directories in both locations. */
|
|
74
|
-
function findSharedSkills(): string[] {
|
|
75
|
-
const embedded = new Set(listSkillDirs(EMBEDDED_SKILLS_DIR));
|
|
76
|
-
const toplevel = new Set(listSkillDirs(TOPLEVEL_SKILLS_DIR));
|
|
77
|
-
return [...embedded].filter((id) => toplevel.has(id)).sort();
|
|
78
|
-
}
|
|
79
|
-
|
|
80
|
-
// ---------------------------------------------------------------------------
|
|
81
|
-
// Tests
|
|
82
|
-
// ---------------------------------------------------------------------------
|
|
83
|
-
|
|
84
|
-
describe('Skill mirror parity — embedded ↔ top-level', () => {
|
|
85
|
-
const sharedSkills = findSharedSkills();
|
|
86
|
-
|
|
87
|
-
test('at least one shared skill exists (sanity check)', () => {
|
|
88
|
-
expect(sharedSkills.length).toBeGreaterThan(0);
|
|
89
|
-
});
|
|
90
|
-
|
|
91
|
-
// ── Every top-level skill must have an embedded source ──────────────────
|
|
92
|
-
|
|
93
|
-
test('every top-level skill directory has a corresponding embedded directory', () => {
|
|
94
|
-
const embedded = new Set(listSkillDirs(EMBEDDED_SKILLS_DIR));
|
|
95
|
-
const toplevel = listSkillDirs(TOPLEVEL_SKILLS_DIR);
|
|
96
|
-
|
|
97
|
-
const missingFromEmbedded = toplevel.filter(
|
|
98
|
-
(id) => !embedded.has(id) && !TOPLEVEL_ONLY_SKILLS.has(id),
|
|
99
|
-
);
|
|
100
|
-
|
|
101
|
-
expect(missingFromEmbedded).toEqual([]);
|
|
102
|
-
});
|
|
103
|
-
|
|
104
|
-
// ── SKILL.md content parity ─────────────────────────────────────────────
|
|
105
|
-
|
|
106
|
-
for (const skillId of sharedSkills) {
|
|
107
|
-
test(`${skillId}/SKILL.md content matches between embedded and top-level`, () => {
|
|
108
|
-
const embeddedPath = join(EMBEDDED_SKILLS_DIR, skillId, 'SKILL.md');
|
|
109
|
-
const toplevelPath = join(TOPLEVEL_SKILLS_DIR, skillId, 'SKILL.md');
|
|
110
|
-
|
|
111
|
-
const embeddedExists = existsSync(embeddedPath);
|
|
112
|
-
const toplevelExists = existsSync(toplevelPath);
|
|
113
|
-
|
|
114
|
-
if (!embeddedExists && !toplevelExists) {
|
|
115
|
-
// Neither has a SKILL.md — acceptable, no parity violation
|
|
116
|
-
return;
|
|
117
|
-
}
|
|
118
|
-
|
|
119
|
-
// If one exists but not the other, that is a drift violation
|
|
120
|
-
expect(embeddedExists).toBe(
|
|
121
|
-
toplevelExists,
|
|
122
|
-
);
|
|
123
|
-
|
|
124
|
-
if (!embeddedExists || !toplevelExists) return;
|
|
125
|
-
|
|
126
|
-
const embeddedContent = readFileSync(embeddedPath, 'utf-8');
|
|
127
|
-
const toplevelContent = readFileSync(toplevelPath, 'utf-8');
|
|
128
|
-
|
|
129
|
-
expect(embeddedContent).toBe(toplevelContent);
|
|
130
|
-
});
|
|
131
|
-
}
|
|
132
|
-
|
|
133
|
-
// ── Every top-level catalog entry must have an embedded source ──────────
|
|
134
|
-
|
|
135
|
-
test('every top-level catalog entry exists in the embedded catalog', () => {
|
|
136
|
-
expect(existsSync(EMBEDDED_CATALOG)).toBe(true);
|
|
137
|
-
expect(existsSync(TOPLEVEL_CATALOG)).toBe(true);
|
|
138
|
-
|
|
139
|
-
const embeddedCatalog: Catalog = JSON.parse(readFileSync(EMBEDDED_CATALOG, 'utf-8'));
|
|
140
|
-
const toplevelCatalog: Catalog = JSON.parse(readFileSync(TOPLEVEL_CATALOG, 'utf-8'));
|
|
141
|
-
|
|
142
|
-
const embeddedIds = new Set(embeddedCatalog.skills.map((s) => s.id));
|
|
143
|
-
const toplevelIds = toplevelCatalog.skills.map((s) => s.id);
|
|
144
|
-
|
|
145
|
-
const missingFromEmbedded = toplevelIds.filter(
|
|
146
|
-
(id) => !embeddedIds.has(id) && !TOPLEVEL_ONLY_SKILLS.has(id),
|
|
147
|
-
);
|
|
148
|
-
|
|
149
|
-
expect(missingFromEmbedded).toEqual([]);
|
|
150
|
-
});
|
|
151
|
-
|
|
152
|
-
// ── Catalog entry parity for shared skills ──────────────────────────────
|
|
153
|
-
|
|
154
|
-
test('catalog.json entries match for all shared skills', () => {
|
|
155
|
-
expect(existsSync(EMBEDDED_CATALOG)).toBe(true);
|
|
156
|
-
expect(existsSync(TOPLEVEL_CATALOG)).toBe(true);
|
|
157
|
-
|
|
158
|
-
const embeddedCatalog: Catalog = JSON.parse(readFileSync(EMBEDDED_CATALOG, 'utf-8'));
|
|
159
|
-
const toplevelCatalog: Catalog = JSON.parse(readFileSync(TOPLEVEL_CATALOG, 'utf-8'));
|
|
160
|
-
|
|
161
|
-
const embeddedMap = new Map(embeddedCatalog.skills.map((s) => [s.id, s]));
|
|
162
|
-
const toplevelMap = new Map(toplevelCatalog.skills.map((s) => [s.id, s]));
|
|
163
|
-
|
|
164
|
-
// Only compare entries that exist in both catalogs
|
|
165
|
-
const sharedIds = [...embeddedMap.keys()].filter((id) => toplevelMap.has(id));
|
|
166
|
-
|
|
167
|
-
expect(sharedIds.length).toBeGreaterThan(0);
|
|
168
|
-
|
|
169
|
-
for (const id of sharedIds) {
|
|
170
|
-
const embedded = embeddedMap.get(id)!;
|
|
171
|
-
const toplevel = toplevelMap.get(id)!;
|
|
172
|
-
|
|
173
|
-
expect(embedded).toEqual(toplevel);
|
|
174
|
-
}
|
|
175
|
-
});
|
|
176
|
-
});
|
|
@@ -1,63 +0,0 @@
|
|
|
1
|
-
{
|
|
2
|
-
"description": "Manifest of first-party Vellum skills. Fetched from GitHub at runtime so the assistant can discover and install new skills maintained by Vellum.",
|
|
3
|
-
"version": 1,
|
|
4
|
-
"skills": [
|
|
5
|
-
{
|
|
6
|
-
"id": "chatgpt-import",
|
|
7
|
-
"name": "ChatGPT Import",
|
|
8
|
-
"description": "Import conversation history from ChatGPT into Vellum",
|
|
9
|
-
"emoji": "\ud83d\udce5"
|
|
10
|
-
},
|
|
11
|
-
{
|
|
12
|
-
"id": "deploy-fullstack-vercel",
|
|
13
|
-
"name": "Deploy Fullstack to Vercel",
|
|
14
|
-
"description": "Build and deploy a full-stack app (React frontend + Python/FastAPI backend) to Vercel as a serverless demo with seeded data",
|
|
15
|
-
"emoji": "\ud83d\ude80"
|
|
16
|
-
},
|
|
17
|
-
{
|
|
18
|
-
"id": "document-writer",
|
|
19
|
-
"name": "Document Writer",
|
|
20
|
-
"description": "Create and edit long-form documents like blog posts, articles, essays, and reports using the built-in rich text editor",
|
|
21
|
-
"emoji": "\ud83d\udcdd"
|
|
22
|
-
},
|
|
23
|
-
{
|
|
24
|
-
"id": "slack-oauth-setup",
|
|
25
|
-
"name": "Slack OAuth Setup",
|
|
26
|
-
"description": "Create Slack App and OAuth credentials for Slack integration using browser automation",
|
|
27
|
-
"emoji": "\ud83d\udd11",
|
|
28
|
-
"includes": ["browser", "public-ingress"]
|
|
29
|
-
},
|
|
30
|
-
{
|
|
31
|
-
"id": "telegram-setup",
|
|
32
|
-
"name": "Telegram Setup",
|
|
33
|
-
"description": "Connect a Telegram bot to the Vellum Assistant gateway with automated webhook registration and credential storage",
|
|
34
|
-
"emoji": "\ud83e\udd16",
|
|
35
|
-
"includes": ["public-ingress"]
|
|
36
|
-
},
|
|
37
|
-
{
|
|
38
|
-
"id": "twilio-setup",
|
|
39
|
-
"name": "Twilio Setup",
|
|
40
|
-
"description": "Configure Twilio credentials and phone numbers for voice calls and SMS messaging",
|
|
41
|
-
"emoji": "\ud83d\udcf1",
|
|
42
|
-
"includes": ["public-ingress"]
|
|
43
|
-
},
|
|
44
|
-
{
|
|
45
|
-
"id": "sms-setup",
|
|
46
|
-
"name": "SMS Setup",
|
|
47
|
-
"description": "Set up and troubleshoot SMS messaging with guided Twilio configuration, compliance, and verification",
|
|
48
|
-
"emoji": "\ud83d\udce8"
|
|
49
|
-
},
|
|
50
|
-
{
|
|
51
|
-
"id": "guardian-verify-setup",
|
|
52
|
-
"name": "Guardian Verify Setup",
|
|
53
|
-
"description": "Set up guardian verification for SMS, voice, or Telegram channels via outbound verification flow",
|
|
54
|
-
"emoji": "\ud83d\udd10"
|
|
55
|
-
},
|
|
56
|
-
{
|
|
57
|
-
"id": "trusted-contacts",
|
|
58
|
-
"name": "Trusted Contacts",
|
|
59
|
-
"description": "Manage trusted contacts and Telegram invite links \u2014 list, allow, revoke, block users, and create/list/revoke shareable invite links",
|
|
60
|
-
"emoji": "\ud83d\udc65"
|
|
61
|
-
}
|
|
62
|
-
]
|
|
63
|
-
}
|
|
@@ -1,295 +0,0 @@
|
|
|
1
|
-
import { existsSync, readFileSync } from 'node:fs';
|
|
2
|
-
|
|
3
|
-
import { eq } from 'drizzle-orm';
|
|
4
|
-
import { v4 as uuid } from 'uuid';
|
|
5
|
-
|
|
6
|
-
import { addMessage, createConversation, setConversationOriginInterfaceIfUnset } from '../../../../memory/conversation-store.js';
|
|
7
|
-
import { getDb } from '../../../../memory/db.js';
|
|
8
|
-
import { conversationKeys,conversations, messages as messagesTable } from '../../../../memory/schema.js';
|
|
9
|
-
import type { ToolContext, ToolExecutionResult } from '../../../../tools/types.js';
|
|
10
|
-
|
|
11
|
-
// -- ChatGPT export format types --
|
|
12
|
-
|
|
13
|
-
interface ChatGPTContent {
|
|
14
|
-
content_type: string;
|
|
15
|
-
parts?: (string | null | Record<string, unknown>)[];
|
|
16
|
-
}
|
|
17
|
-
|
|
18
|
-
interface ChatGPTNode {
|
|
19
|
-
message: {
|
|
20
|
-
author: { role: string };
|
|
21
|
-
content: ChatGPTContent;
|
|
22
|
-
create_time?: number | null;
|
|
23
|
-
} | null;
|
|
24
|
-
parent: string | null;
|
|
25
|
-
children: string[];
|
|
26
|
-
}
|
|
27
|
-
|
|
28
|
-
interface ChatGPTConversation {
|
|
29
|
-
id?: string;
|
|
30
|
-
title: string;
|
|
31
|
-
create_time: number;
|
|
32
|
-
update_time: number;
|
|
33
|
-
current_node: string;
|
|
34
|
-
mapping: Record<string, ChatGPTNode>;
|
|
35
|
-
}
|
|
36
|
-
|
|
37
|
-
interface ImportedMessage {
|
|
38
|
-
role: string;
|
|
39
|
-
content: Array<{ type: string; text: string }>;
|
|
40
|
-
createdAt: number;
|
|
41
|
-
}
|
|
42
|
-
|
|
43
|
-
interface ImportedConversation {
|
|
44
|
-
sourceId: string;
|
|
45
|
-
title: string;
|
|
46
|
-
createdAt: number;
|
|
47
|
-
updatedAt: number;
|
|
48
|
-
messages: ImportedMessage[];
|
|
49
|
-
}
|
|
50
|
-
|
|
51
|
-
// -- Tool entry point --
|
|
52
|
-
|
|
53
|
-
export async function run(
|
|
54
|
-
input: Record<string, unknown>,
|
|
55
|
-
_context: ToolContext,
|
|
56
|
-
): Promise<ToolExecutionResult> {
|
|
57
|
-
const filePath = input.file_path as string;
|
|
58
|
-
|
|
59
|
-
if (!filePath) {
|
|
60
|
-
return { content: 'Error: file_path is required', isError: true };
|
|
61
|
-
}
|
|
62
|
-
|
|
63
|
-
if (!filePath.endsWith('.zip')) {
|
|
64
|
-
return { content: 'Error: Only ZIP files are accepted. Please provide the ChatGPT export ZIP file.', isError: true };
|
|
65
|
-
}
|
|
66
|
-
|
|
67
|
-
if (!existsSync(filePath)) {
|
|
68
|
-
return { content: `Error: File not found: ${filePath}`, isError: true };
|
|
69
|
-
}
|
|
70
|
-
|
|
71
|
-
let imported: ImportedConversation[];
|
|
72
|
-
try {
|
|
73
|
-
imported = parseChatGPTExport(filePath);
|
|
74
|
-
} catch (err) {
|
|
75
|
-
return {
|
|
76
|
-
content: `Error parsing export file: ${err instanceof Error ? err.message : String(err)}`,
|
|
77
|
-
isError: true,
|
|
78
|
-
};
|
|
79
|
-
}
|
|
80
|
-
|
|
81
|
-
if (imported.length === 0) {
|
|
82
|
-
return { content: 'No conversations found in the export file.', isError: false };
|
|
83
|
-
}
|
|
84
|
-
|
|
85
|
-
const db = getDb();
|
|
86
|
-
let importedCount = 0;
|
|
87
|
-
let skippedCount = 0;
|
|
88
|
-
let messageCount = 0;
|
|
89
|
-
|
|
90
|
-
for (const conv of imported) {
|
|
91
|
-
const convKey = `chatgpt:${conv.sourceId}`;
|
|
92
|
-
|
|
93
|
-
const existing = db
|
|
94
|
-
.select()
|
|
95
|
-
.from(conversationKeys)
|
|
96
|
-
.where(eq(conversationKeys.conversationKey, convKey))
|
|
97
|
-
.get();
|
|
98
|
-
|
|
99
|
-
if (existing) {
|
|
100
|
-
skippedCount++;
|
|
101
|
-
continue;
|
|
102
|
-
}
|
|
103
|
-
|
|
104
|
-
const conversation = createConversation(conv.title);
|
|
105
|
-
const importChannelMetadata = {
|
|
106
|
-
userMessageChannel: 'vellum',
|
|
107
|
-
assistantMessageChannel: 'vellum',
|
|
108
|
-
userMessageInterface: 'vellum',
|
|
109
|
-
assistantMessageInterface: 'vellum',
|
|
110
|
-
} as const;
|
|
111
|
-
|
|
112
|
-
for (const msg of conv.messages) {
|
|
113
|
-
await addMessage(conversation.id, msg.role, JSON.stringify(msg.content), importChannelMetadata);
|
|
114
|
-
}
|
|
115
|
-
|
|
116
|
-
// addMessage auto-fills originChannel but not originInterface, so set it explicitly
|
|
117
|
-
setConversationOriginInterfaceIfUnset(conversation.id, 'vellum');
|
|
118
|
-
|
|
119
|
-
// Override timestamps to match ChatGPT originals
|
|
120
|
-
db.update(conversations)
|
|
121
|
-
.set({ createdAt: conv.createdAt, updatedAt: conv.updatedAt })
|
|
122
|
-
.where(eq(conversations.id, conversation.id))
|
|
123
|
-
.run();
|
|
124
|
-
|
|
125
|
-
// Update message timestamps to match ChatGPT originals
|
|
126
|
-
const dbMessages = db
|
|
127
|
-
.select({ id: messagesTable.id })
|
|
128
|
-
.from(messagesTable)
|
|
129
|
-
.where(eq(messagesTable.conversationId, conversation.id))
|
|
130
|
-
.orderBy(messagesTable.createdAt)
|
|
131
|
-
.all();
|
|
132
|
-
|
|
133
|
-
for (let i = 0; i < dbMessages.length && i < conv.messages.length; i++) {
|
|
134
|
-
db.update(messagesTable)
|
|
135
|
-
.set({ createdAt: conv.messages[i].createdAt })
|
|
136
|
-
.where(eq(messagesTable.id, dbMessages[i].id))
|
|
137
|
-
.run();
|
|
138
|
-
}
|
|
139
|
-
|
|
140
|
-
db.insert(conversationKeys)
|
|
141
|
-
.values({
|
|
142
|
-
id: uuid(),
|
|
143
|
-
conversationKey: convKey,
|
|
144
|
-
conversationId: conversation.id,
|
|
145
|
-
createdAt: Date.now(),
|
|
146
|
-
})
|
|
147
|
-
.run();
|
|
148
|
-
|
|
149
|
-
importedCount++;
|
|
150
|
-
messageCount += conv.messages.length;
|
|
151
|
-
}
|
|
152
|
-
|
|
153
|
-
const lines = [`Imported ${importedCount} conversation(s) with ${messageCount} message(s).`];
|
|
154
|
-
if (skippedCount > 0) {
|
|
155
|
-
lines.push(`Skipped ${skippedCount} already-imported conversation(s).`);
|
|
156
|
-
}
|
|
157
|
-
return { content: lines.join('\n'), isError: false };
|
|
158
|
-
}
|
|
159
|
-
|
|
160
|
-
// -- Parser --
|
|
161
|
-
|
|
162
|
-
function parseChatGPTExport(zipPath: string): ImportedConversation[] {
|
|
163
|
-
const jsonContent = extractConversationsJsonFromZip(zipPath);
|
|
164
|
-
|
|
165
|
-
const raw = JSON.parse(jsonContent);
|
|
166
|
-
if (!Array.isArray(raw)) {
|
|
167
|
-
throw new Error('Expected conversations.json to contain a JSON array');
|
|
168
|
-
}
|
|
169
|
-
|
|
170
|
-
const results: ImportedConversation[] = [];
|
|
171
|
-
for (const conv of raw as ChatGPTConversation[]) {
|
|
172
|
-
const imported = parseConversation(conv);
|
|
173
|
-
if (imported) {
|
|
174
|
-
results.push(imported);
|
|
175
|
-
}
|
|
176
|
-
}
|
|
177
|
-
return results;
|
|
178
|
-
}
|
|
179
|
-
|
|
180
|
-
function parseConversation(conv: ChatGPTConversation): ImportedConversation | null {
|
|
181
|
-
const { mapping, current_node } = conv;
|
|
182
|
-
if (!mapping || !current_node || !mapping[current_node]) return null;
|
|
183
|
-
|
|
184
|
-
// Walk from current_node to root via parent pointers, then reverse for chronological order
|
|
185
|
-
const nodeIds: string[] = [];
|
|
186
|
-
let nodeId: string | null = current_node;
|
|
187
|
-
while (nodeId) {
|
|
188
|
-
nodeIds.push(nodeId);
|
|
189
|
-
nodeId = mapping[nodeId]?.parent ?? null;
|
|
190
|
-
}
|
|
191
|
-
nodeIds.reverse();
|
|
192
|
-
|
|
193
|
-
const messages: ImportedMessage[] = [];
|
|
194
|
-
for (const id of nodeIds) {
|
|
195
|
-
const node = mapping[id];
|
|
196
|
-
if (!node?.message) continue;
|
|
197
|
-
|
|
198
|
-
const { author, content, create_time } = node.message;
|
|
199
|
-
const role = author?.role;
|
|
200
|
-
if (role !== 'user' && role !== 'assistant') continue;
|
|
201
|
-
|
|
202
|
-
const text = extractText(content);
|
|
203
|
-
if (!text) continue;
|
|
204
|
-
|
|
205
|
-
messages.push({
|
|
206
|
-
role,
|
|
207
|
-
content: [{ type: 'text', text }],
|
|
208
|
-
createdAt: create_time ? Math.round(create_time * 1000) : Math.round(conv.create_time * 1000),
|
|
209
|
-
});
|
|
210
|
-
}
|
|
211
|
-
|
|
212
|
-
if (messages.length === 0) return null;
|
|
213
|
-
|
|
214
|
-
return {
|
|
215
|
-
sourceId: conv.id ?? `${conv.title}-${conv.create_time}`,
|
|
216
|
-
title: conv.title || 'Untitled',
|
|
217
|
-
createdAt: Math.round(conv.create_time * 1000),
|
|
218
|
-
updatedAt: Math.round(conv.update_time * 1000),
|
|
219
|
-
messages,
|
|
220
|
-
};
|
|
221
|
-
}
|
|
222
|
-
|
|
223
|
-
function extractText(content: ChatGPTContent): string {
|
|
224
|
-
if (!content?.parts) return '';
|
|
225
|
-
return content.parts.filter((p): p is string => typeof p === 'string').join('');
|
|
226
|
-
}
|
|
227
|
-
|
|
228
|
-
// -- ZIP extraction --
|
|
229
|
-
|
|
230
|
-
function extractConversationsJsonFromZip(zipPath: string): string {
|
|
231
|
-
const buffer = readFileSync(zipPath);
|
|
232
|
-
|
|
233
|
-
// Find end of central directory record (EOCD signature: 0x06054b50)
|
|
234
|
-
let eocdOffset = -1;
|
|
235
|
-
for (let i = buffer.length - 22; i >= 0; i--) {
|
|
236
|
-
if (buffer[i] === 0x50 && buffer[i + 1] === 0x4b && buffer[i + 2] === 0x05 && buffer[i + 3] === 0x06) {
|
|
237
|
-
eocdOffset = i;
|
|
238
|
-
break;
|
|
239
|
-
}
|
|
240
|
-
}
|
|
241
|
-
if (eocdOffset === -1) {
|
|
242
|
-
throw new Error('Invalid ZIP file: could not find end of central directory');
|
|
243
|
-
}
|
|
244
|
-
|
|
245
|
-
const centralDirOffset = buffer.readUInt32LE(eocdOffset + 16);
|
|
246
|
-
const centralDirEntries = buffer.readUInt16LE(eocdOffset + 10);
|
|
247
|
-
|
|
248
|
-
// Walk central directory to find conversations.json
|
|
249
|
-
let offset = centralDirOffset;
|
|
250
|
-
for (let i = 0; i < centralDirEntries; i++) {
|
|
251
|
-
if (buffer[offset] !== 0x50 || buffer[offset + 1] !== 0x4b || buffer[offset + 2] !== 0x01 || buffer[offset + 3] !== 0x02) {
|
|
252
|
-
throw new Error('Invalid ZIP central directory entry');
|
|
253
|
-
}
|
|
254
|
-
|
|
255
|
-
const cdCompressedSize = buffer.readUInt32LE(offset + 20);
|
|
256
|
-
const fileNameLength = buffer.readUInt16LE(offset + 28);
|
|
257
|
-
const extraLength = buffer.readUInt16LE(offset + 30);
|
|
258
|
-
const commentLength = buffer.readUInt16LE(offset + 32);
|
|
259
|
-
const localHeaderOffset = buffer.readUInt32LE(offset + 42);
|
|
260
|
-
const fileName = buffer.subarray(offset + 46, offset + 46 + fileNameLength).toString('utf-8');
|
|
261
|
-
|
|
262
|
-
if (fileName === 'conversations.json' || fileName.endsWith('/conversations.json')) {
|
|
263
|
-
return extractLocalFile(buffer, localHeaderOffset, cdCompressedSize);
|
|
264
|
-
}
|
|
265
|
-
|
|
266
|
-
offset += 46 + fileNameLength + extraLength + commentLength;
|
|
267
|
-
}
|
|
268
|
-
|
|
269
|
-
throw new Error('conversations.json not found in ZIP file');
|
|
270
|
-
}
|
|
271
|
-
|
|
272
|
-
function extractLocalFile(buffer: Buffer, offset: number, cdCompressedSize: number): string {
|
|
273
|
-
if (buffer[offset] !== 0x50 || buffer[offset + 1] !== 0x4b || buffer[offset + 2] !== 0x03 || buffer[offset + 3] !== 0x04) {
|
|
274
|
-
throw new Error('Invalid ZIP local file header');
|
|
275
|
-
}
|
|
276
|
-
|
|
277
|
-
const compressionMethod = buffer.readUInt16LE(offset + 8);
|
|
278
|
-
const localCompressedSize = buffer.readUInt32LE(offset + 18);
|
|
279
|
-
const compressedSize = cdCompressedSize > 0 ? cdCompressedSize : localCompressedSize;
|
|
280
|
-
const fileNameLength = buffer.readUInt16LE(offset + 26);
|
|
281
|
-
const extraLength = buffer.readUInt16LE(offset + 28);
|
|
282
|
-
|
|
283
|
-
const dataOffset = offset + 30 + fileNameLength + extraLength;
|
|
284
|
-
const fileData = buffer.subarray(dataOffset, dataOffset + compressedSize);
|
|
285
|
-
|
|
286
|
-
if (compressionMethod === 0) {
|
|
287
|
-
return fileData.toString('utf-8');
|
|
288
|
-
} else if (compressionMethod === 8) {
|
|
289
|
-
// eslint-disable-next-line @typescript-eslint/no-require-imports
|
|
290
|
-
const { inflateRawSync } = require('node:zlib') as typeof import('node:zlib');
|
|
291
|
-
return inflateRawSync(fileData).toString('utf-8');
|
|
292
|
-
} else {
|
|
293
|
-
throw new Error(`Unsupported ZIP compression method: ${compressionMethod}`);
|
|
294
|
-
}
|
|
295
|
-
}
|
|
@@ -1,166 +0,0 @@
|
|
|
1
|
-
import { readFileSync } from 'node:fs';
|
|
2
|
-
import { join } from 'node:path';
|
|
3
|
-
import { gunzipSync } from 'node:zlib';
|
|
4
|
-
|
|
5
|
-
import type { CatalogEntry } from '../tools/skills/vellum-catalog.js';
|
|
6
|
-
import { getLogger } from '../util/logger.js';
|
|
7
|
-
import { readPlatformToken } from '../util/platform.js';
|
|
8
|
-
|
|
9
|
-
const log = getLogger('vellum-catalog-remote');
|
|
10
|
-
|
|
11
|
-
const PLATFORM_URL = process.env.VELLUM_ASSISTANT_PLATFORM_URL ?? 'https://assistant.vellum.ai';
|
|
12
|
-
|
|
13
|
-
const CACHE_TTL_MS = 60 * 60 * 1000; // 1 hour
|
|
14
|
-
|
|
15
|
-
interface CatalogManifest {
|
|
16
|
-
version: number;
|
|
17
|
-
skills: CatalogEntry[];
|
|
18
|
-
}
|
|
19
|
-
|
|
20
|
-
let cachedEntries: CatalogEntry[] | null = null;
|
|
21
|
-
let cacheTimestamp = 0;
|
|
22
|
-
|
|
23
|
-
function getBundledCatalogPath(): string {
|
|
24
|
-
return join(import.meta.dir, '..', 'config', 'vellum-skills', 'catalog.json');
|
|
25
|
-
}
|
|
26
|
-
|
|
27
|
-
function loadBundledCatalog(): CatalogEntry[] {
|
|
28
|
-
try {
|
|
29
|
-
const raw = readFileSync(getBundledCatalogPath(), 'utf-8');
|
|
30
|
-
const manifest: CatalogManifest = JSON.parse(raw);
|
|
31
|
-
return manifest.skills ?? [];
|
|
32
|
-
} catch (err) {
|
|
33
|
-
log.warn({ err }, 'Failed to read bundled catalog.json');
|
|
34
|
-
return [];
|
|
35
|
-
}
|
|
36
|
-
}
|
|
37
|
-
|
|
38
|
-
function getBundledSkillContent(skillId: string): string | null {
|
|
39
|
-
try {
|
|
40
|
-
const skillPath = join(import.meta.dir, '..', 'config', 'vellum-skills', skillId, 'SKILL.md');
|
|
41
|
-
return readFileSync(skillPath, 'utf-8');
|
|
42
|
-
} catch {
|
|
43
|
-
return null;
|
|
44
|
-
}
|
|
45
|
-
}
|
|
46
|
-
|
|
47
|
-
/** Build request headers, including platform token when available. */
|
|
48
|
-
function buildPlatformHeaders(): Record<string, string> {
|
|
49
|
-
const headers: Record<string, string> = {};
|
|
50
|
-
const token = readPlatformToken();
|
|
51
|
-
if (token) {
|
|
52
|
-
headers['X-Session-Token'] = token;
|
|
53
|
-
}
|
|
54
|
-
return headers;
|
|
55
|
-
}
|
|
56
|
-
|
|
57
|
-
/**
|
|
58
|
-
* Fetch catalog entries from the platform API. Falls back to bundled copy.
|
|
59
|
-
* Reads the platform token from ~/.vellum/platform-token automatically.
|
|
60
|
-
*/
|
|
61
|
-
export async function fetchCatalogEntries(): Promise<CatalogEntry[]> {
|
|
62
|
-
const now = Date.now();
|
|
63
|
-
if (cachedEntries && now - cacheTimestamp < CACHE_TTL_MS) {
|
|
64
|
-
return cachedEntries;
|
|
65
|
-
}
|
|
66
|
-
|
|
67
|
-
try {
|
|
68
|
-
const url = `${PLATFORM_URL}/v1/skills/`;
|
|
69
|
-
const response = await fetch(url, {
|
|
70
|
-
headers: buildPlatformHeaders(),
|
|
71
|
-
signal: AbortSignal.timeout(5000),
|
|
72
|
-
});
|
|
73
|
-
|
|
74
|
-
if (!response.ok) {
|
|
75
|
-
throw new Error(`HTTP ${response.status}: ${response.statusText}`);
|
|
76
|
-
}
|
|
77
|
-
|
|
78
|
-
const manifest: CatalogManifest = await response.json();
|
|
79
|
-
const skills = manifest.skills;
|
|
80
|
-
if (!Array.isArray(skills) || skills.length === 0) {
|
|
81
|
-
throw new Error('Platform catalog has invalid or empty skills array');
|
|
82
|
-
}
|
|
83
|
-
cachedEntries = skills;
|
|
84
|
-
cacheTimestamp = now;
|
|
85
|
-
log.info({ count: cachedEntries.length }, 'Fetched vellum-skills catalog from platform API');
|
|
86
|
-
return cachedEntries;
|
|
87
|
-
} catch (err) {
|
|
88
|
-
log.warn({ err }, 'Failed to fetch catalog from platform API, falling back to bundled copy');
|
|
89
|
-
const bundled = loadBundledCatalog();
|
|
90
|
-
// Cache the bundled result too so we don't re-fetch on every call during outage
|
|
91
|
-
cachedEntries = bundled;
|
|
92
|
-
cacheTimestamp = now;
|
|
93
|
-
return bundled;
|
|
94
|
-
}
|
|
95
|
-
}
|
|
96
|
-
|
|
97
|
-
/**
|
|
98
|
-
* Extract SKILL.md content from a tar archive (uncompressed).
|
|
99
|
-
* Tar format: 512-byte header blocks followed by file data blocks.
|
|
100
|
-
*/
|
|
101
|
-
function extractSkillMdFromTar(tarBuffer: Buffer): string | null {
|
|
102
|
-
let offset = 0;
|
|
103
|
-
while (offset + 512 <= tarBuffer.length) {
|
|
104
|
-
const header = tarBuffer.subarray(offset, offset + 512);
|
|
105
|
-
|
|
106
|
-
// Check for end-of-archive (two consecutive zero blocks)
|
|
107
|
-
if (header.every((b) => b === 0)) break;
|
|
108
|
-
|
|
109
|
-
// Extract filename (bytes 0-99, null-terminated)
|
|
110
|
-
const nameEnd = header.indexOf(0, 0);
|
|
111
|
-
const name = header.subarray(0, Math.min(nameEnd >= 0 ? nameEnd : 100, 100)).toString('utf-8');
|
|
112
|
-
|
|
113
|
-
// Extract file size (bytes 124-135, octal string)
|
|
114
|
-
const sizeStr = header.subarray(124, 136).toString('utf-8').trim();
|
|
115
|
-
const size = parseInt(sizeStr, 8) || 0;
|
|
116
|
-
|
|
117
|
-
offset += 512; // move past header
|
|
118
|
-
|
|
119
|
-
if (name.endsWith('SKILL.md') || name === 'SKILL.md') {
|
|
120
|
-
return tarBuffer.subarray(offset, offset + size).toString('utf-8');
|
|
121
|
-
}
|
|
122
|
-
|
|
123
|
-
// Skip to next header (data blocks are padded to 512 bytes)
|
|
124
|
-
offset += Math.ceil(size / 512) * 512;
|
|
125
|
-
}
|
|
126
|
-
return null;
|
|
127
|
-
}
|
|
128
|
-
|
|
129
|
-
/**
|
|
130
|
-
* Fetch a skill's SKILL.md content from the platform tar API.
|
|
131
|
-
* GET /v1/skills/{skill_id}/ returns a tar.gz archive containing all skill files.
|
|
132
|
-
* Falls back to bundled copy on failure.
|
|
133
|
-
*/
|
|
134
|
-
export async function fetchSkillContent(skillId: string): Promise<string | null> {
|
|
135
|
-
try {
|
|
136
|
-
const url = `${PLATFORM_URL}/v1/skills/${encodeURIComponent(skillId)}/`;
|
|
137
|
-
const response = await fetch(url, {
|
|
138
|
-
headers: buildPlatformHeaders(),
|
|
139
|
-
signal: AbortSignal.timeout(15000),
|
|
140
|
-
});
|
|
141
|
-
|
|
142
|
-
if (!response.ok) {
|
|
143
|
-
throw new Error(`HTTP ${response.status}: ${response.statusText}`);
|
|
144
|
-
}
|
|
145
|
-
|
|
146
|
-
const gzipBuffer = Buffer.from(await response.arrayBuffer());
|
|
147
|
-
const tarBuffer = gunzipSync(gzipBuffer);
|
|
148
|
-
const skillMd = extractSkillMdFromTar(tarBuffer);
|
|
149
|
-
|
|
150
|
-
if (skillMd) {
|
|
151
|
-
return skillMd;
|
|
152
|
-
}
|
|
153
|
-
|
|
154
|
-
log.warn({ skillId }, 'SKILL.md not found in platform tar archive, falling back to bundled');
|
|
155
|
-
} catch (err) {
|
|
156
|
-
log.warn({ err, skillId }, 'Failed to fetch skill content from platform API, falling back to bundled');
|
|
157
|
-
}
|
|
158
|
-
|
|
159
|
-
return getBundledSkillContent(skillId);
|
|
160
|
-
}
|
|
161
|
-
|
|
162
|
-
/** Check if a skill ID exists in the catalog. */
|
|
163
|
-
export async function checkVellumSkill(skillId: string): Promise<boolean> {
|
|
164
|
-
const entries = await fetchCatalogEntries();
|
|
165
|
-
return entries.some((e) => e.id === skillId);
|
|
166
|
-
}
|