@aion0/forge 0.8.1 → 0.8.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/RELEASE_NOTES.md +6 -6
- package/app/api/connectors/[id]/settings/route.ts +31 -37
- package/app/api/connectors/[id]/test/route.ts +260 -0
- package/app/api/connectors/install-local/route.ts +211 -0
- package/app/api/connectors/marketplace/route.ts +79 -0
- package/app/api/connectors/route.ts +41 -46
- package/app/api/jobs/route.ts +4 -0
- package/app/api/skills/install-local/route.ts +282 -0
- package/components/ConnectorsPanel.tsx +526 -211
- package/components/SettingsModal.tsx +1 -0
- package/components/SkillsPanel.tsx +42 -1
- package/lib/agents/claude-adapter.ts +4 -0
- package/lib/agents/types.ts +6 -0
- package/lib/chat/agent-loop.ts +13 -22
- package/lib/chat/protocols/http.ts +1 -1
- package/lib/chat/protocols/shell.ts +1 -1
- package/lib/chat/tool-dispatcher.ts +20 -20
- package/lib/connectors/migration.ts +110 -0
- package/lib/connectors/registry.ts +328 -0
- package/lib/connectors/sync.ts +305 -0
- package/lib/connectors/types.ts +253 -0
- package/lib/help-docs/00-overview.md +1 -0
- package/lib/help-docs/17-connectors.md +241 -189
- package/lib/help-docs/21-build-connector.md +314 -0
- package/lib/help-docs/CLAUDE.md +4 -2
- package/lib/init.ts +25 -0
- package/lib/jobs/dispatcher.ts +28 -8
- package/lib/jobs/scheduler.ts +66 -6
- package/lib/jobs/store.ts +51 -2
- package/lib/jobs/types.ts +32 -0
- package/lib/pipeline-scheduler.ts +3 -2
- package/lib/pipeline.ts +137 -15
- package/lib/plugins/registry.ts +9 -42
- package/lib/plugins/types.ts +4 -129
- package/lib/settings.ts +7 -0
- package/lib/skills.ts +27 -1
- package/lib/task-manager.ts +62 -2
- package/package.json +4 -1
- package/src/core/db/database.ts +4 -0
- package/lib/builtin-plugins/github-api.yaml +0 -93
- package/lib/builtin-plugins/gitlab.yaml +0 -860
- package/lib/builtin-plugins/mantis.probe.js +0 -176
- package/lib/builtin-plugins/mantis.yaml +0 -964
- package/lib/builtin-plugins/pmdb.yaml +0 -178
- package/lib/builtin-plugins/teams.yaml +0 -913
|
@@ -0,0 +1,314 @@
|
|
|
1
|
+
# Build Your Own Connector
|
|
2
|
+
|
|
3
|
+
This doc tells the Forge Help AI how to author a custom connector for
|
|
4
|
+
the user. The user asks something like _"build me a connector for
|
|
5
|
+
JIRA"_ or _"add a connector that scrapes my internal wiki"_; you walk
|
|
6
|
+
them through requirements, generate a manifest, and install it locally.
|
|
7
|
+
|
|
8
|
+
## Mental model
|
|
9
|
+
|
|
10
|
+
A connector is a YAML manifest that exposes **tools** the chat LLM
|
|
11
|
+
can call. Each tool runs in one of three places:
|
|
12
|
+
|
|
13
|
+
| `protocol` | Where it runs | Used for |
|
|
14
|
+
|---|---|---|
|
|
15
|
+
| `browser` (default) | User's tab, via the Forge browser extension | DOM-scraping the user's logged-in UI (Mantis, GitLab UI, Teams) |
|
|
16
|
+
| `http` | Forge server | Clean REST APIs with PAT auth (GitHub, Stripe, …) |
|
|
17
|
+
| `shell` | Forge server | Local CLI tools (`git`, `gh`, `kubectl`, …) |
|
|
18
|
+
|
|
19
|
+
Choose **browser** when the user is already logged in to a web app and
|
|
20
|
+
you don't want to manage tokens. Choose **http** when there's a clean
|
|
21
|
+
REST API. Choose **shell** when wrapping a local CLI.
|
|
22
|
+
|
|
23
|
+
## The interview (ask the user)
|
|
24
|
+
|
|
25
|
+
Before writing anything, ask:
|
|
26
|
+
|
|
27
|
+
1. **What service / site?** (e.g. JIRA Server at jira.acme.com)
|
|
28
|
+
2. **What actions are needed?** Read-only? Read + write? List of verbs.
|
|
29
|
+
3. **Auth?** Logged-in browser session, PAT, OAuth, local CLI?
|
|
30
|
+
4. **What settings does the user need to fill in?** (base_url, PAT, default project, …)
|
|
31
|
+
|
|
32
|
+
Stop after this; show the user a one-paragraph plan; let them adjust.
|
|
33
|
+
|
|
34
|
+
## Manifest shape
|
|
35
|
+
|
|
36
|
+
Required fields: `id`, `name`, `version`, and either `tools` or
|
|
37
|
+
`connectors[]`. Everything else is optional.
|
|
38
|
+
|
|
39
|
+
```yaml
|
|
40
|
+
id: jira # lowercase, alphanumerics + - / _
|
|
41
|
+
name: JIRA
|
|
42
|
+
icon: "📋"
|
|
43
|
+
version: "0.1.0"
|
|
44
|
+
author: "<user-provided>"
|
|
45
|
+
description: |
|
|
46
|
+
Multi-line description shown in the marketplace.
|
|
47
|
+
|
|
48
|
+
# Optional. Locks install to Forge versions that have the required runtime features.
|
|
49
|
+
min_forge_version: "0.8.0"
|
|
50
|
+
|
|
51
|
+
# Browser runner. Default 'main'. Use 'isolated' for strict-CSP
|
|
52
|
+
# sites (Teams, github.com) that block eval in the page world.
|
|
53
|
+
runner: main
|
|
54
|
+
|
|
55
|
+
# Per-user settings rendered as a form in Settings → Connectors.
|
|
56
|
+
settings:
|
|
57
|
+
base_url:
|
|
58
|
+
type: string
|
|
59
|
+
label: JIRA base URL
|
|
60
|
+
required: true
|
|
61
|
+
token:
|
|
62
|
+
type: secret # encrypted at rest (AES-256-GCM)
|
|
63
|
+
label: Personal access token
|
|
64
|
+
description: Create at <base_url>/secure/ViewProfile.jspa
|
|
65
|
+
|
|
66
|
+
# Where the extension finds the authenticated tab. {settings.*} expanded server-side.
|
|
67
|
+
host_match: "{base_url}/*"
|
|
68
|
+
|
|
69
|
+
# Substring detected after navigation → "user not logged in".
|
|
70
|
+
login_redirect: "/login.jsp"
|
|
71
|
+
|
|
72
|
+
tools:
|
|
73
|
+
list_my_issues:
|
|
74
|
+
description: List JIRA issues assigned to me.
|
|
75
|
+
parameters:
|
|
76
|
+
project: { type: string, description: "Limit to one project key (optional)" }
|
|
77
|
+
status: { type: select, options: ["open","done","all"], default: "open" }
|
|
78
|
+
limit: { type: number, default: 25 }
|
|
79
|
+
# protocol omitted → browser
|
|
80
|
+
page:
|
|
81
|
+
url: "{base_url}/issues/?jql=assignee=currentUser()"
|
|
82
|
+
on_target: "/issues/" # skip navigation if URL already matches
|
|
83
|
+
script: |
|
|
84
|
+
const rows = Array.from(document.querySelectorAll('.issuerow'));
|
|
85
|
+
return rows.slice(0, args.limit).map(r => ({
|
|
86
|
+
key: r.dataset.issuekey,
|
|
87
|
+
title: r.querySelector('.summary')?.textContent?.trim(),
|
|
88
|
+
status: r.querySelector('.status')?.textContent?.trim(),
|
|
89
|
+
link: r.querySelector('a.issue-link')?.href,
|
|
90
|
+
}));
|
|
91
|
+
|
|
92
|
+
add_comment:
|
|
93
|
+
destructive: true # extension prompts before running
|
|
94
|
+
description: Add a comment to a JIRA issue.
|
|
95
|
+
parameters:
|
|
96
|
+
issue_key: { type: string, required: true }
|
|
97
|
+
text: { type: string, required: true }
|
|
98
|
+
page:
|
|
99
|
+
url: "{base_url}/browse/{args.issue_key}"
|
|
100
|
+
script: |
|
|
101
|
+
// Same-origin fetch with user's session cookie auto-attached
|
|
102
|
+
const r = await fetch(`/rest/api/2/issue/${args.issue_key}/comment`, {
|
|
103
|
+
method: 'POST',
|
|
104
|
+
headers: { 'Content-Type': 'application/json' },
|
|
105
|
+
body: JSON.stringify({ body: args.text }),
|
|
106
|
+
});
|
|
107
|
+
if (!r.ok) return { _error: `HTTP ${r.status}: ${await r.text()}` };
|
|
108
|
+
return { ok: true };
|
|
109
|
+
```
|
|
110
|
+
|
|
111
|
+
### Template tokens
|
|
112
|
+
|
|
113
|
+
- `{base_url}`, `{settings.<name>}` — expanded server-side at API
|
|
114
|
+
response time from the user's saved settings.
|
|
115
|
+
- `{args.<name>}` — expanded at runtime from the LLM's tool input.
|
|
116
|
+
In a `script` body, prefer `args.foo` (a JS identifier) over the
|
|
117
|
+
literal `{args.foo}` string.
|
|
118
|
+
|
|
119
|
+
### `script` contract
|
|
120
|
+
|
|
121
|
+
- Receives `args` (the LLM's parsed parameters).
|
|
122
|
+
- Returns a JSON-serialisable value (no DOM nodes, no functions).
|
|
123
|
+
- Has access to `document`, `fetch`, `URL`, `Headers`, etc. (page context).
|
|
124
|
+
- Same-origin `fetch()` — the user's session cookies attach automatically.
|
|
125
|
+
- Must be **self-contained** — no closures over Forge / extension code,
|
|
126
|
+
no `import`, no `require`.
|
|
127
|
+
- Throws are caught by the runner and surfaced as tool errors;
|
|
128
|
+
prefer `return { _error: '...' }` for predictable failure messages.
|
|
129
|
+
|
|
130
|
+
### http protocol
|
|
131
|
+
|
|
132
|
+
For services with a clean REST API:
|
|
133
|
+
|
|
134
|
+
```yaml
|
|
135
|
+
tools:
|
|
136
|
+
get_repo:
|
|
137
|
+
protocol: http
|
|
138
|
+
parameters:
|
|
139
|
+
owner: { type: string, required: true }
|
|
140
|
+
repo: { type: string, required: true }
|
|
141
|
+
request:
|
|
142
|
+
method: GET
|
|
143
|
+
url: "https://api.github.com/repos/{args.owner}/{args.repo}"
|
|
144
|
+
headers:
|
|
145
|
+
Accept: "application/vnd.github+json"
|
|
146
|
+
Authorization: "Bearer {settings.token}"
|
|
147
|
+
timeout_ms: 15000
|
|
148
|
+
```
|
|
149
|
+
|
|
150
|
+
### test block
|
|
151
|
+
|
|
152
|
+
A connector can ship a self-test so the Settings → Connectors UI's
|
|
153
|
+
**Test** button has something to call. Two probe kinds:
|
|
154
|
+
|
|
155
|
+
**`probe: http`** (default) — server issues a one-shot HTTP request
|
|
156
|
+
with `{settings.*}` expanded. Use for REST-API connectors with a
|
|
157
|
+
quick `/me`-style endpoint.
|
|
158
|
+
|
|
159
|
+
```yaml
|
|
160
|
+
test:
|
|
161
|
+
description: "GET /api/v4/user — verifies token works"
|
|
162
|
+
probe: http
|
|
163
|
+
request:
|
|
164
|
+
method: GET
|
|
165
|
+
url: "{settings.base_url}/api/v4/user"
|
|
166
|
+
headers:
|
|
167
|
+
PRIVATE-TOKEN: "{settings.token}"
|
|
168
|
+
ok_status: [200] # default [200]
|
|
169
|
+
ok_template: "Authenticated as {{username}} ({{name}})"
|
|
170
|
+
timeout_ms: 15000 # default 15s
|
|
171
|
+
```
|
|
172
|
+
|
|
173
|
+
`ok_template` accepts `{{<json-path>}}` placeholders that resolve
|
|
174
|
+
against the parsed response body. Missing paths render `?`.
|
|
175
|
+
|
|
176
|
+
**`probe: browser`** — forwarded to the paired Forge browser
|
|
177
|
+
extension. The extension opens / acquires a tab matching
|
|
178
|
+
`host_match`, waits for navigation, and reports whether the final
|
|
179
|
+
URL contains `login_redirect`. Use for browser-side connectors
|
|
180
|
+
(Mantis, Teams, PMDB) where auth is the user's session cookie
|
|
181
|
+
rather than a token Forge can verify server-side.
|
|
182
|
+
|
|
183
|
+
```yaml
|
|
184
|
+
test:
|
|
185
|
+
description: "Opens a Mantis tab and checks the session is alive."
|
|
186
|
+
probe: browser
|
|
187
|
+
timeout_ms: 30000 # default 30s
|
|
188
|
+
```
|
|
189
|
+
|
|
190
|
+
No `request:` needed — the probe reuses the manifest's top-level
|
|
191
|
+
`host_match` + `login_redirect`.
|
|
192
|
+
|
|
193
|
+
**Extension wire contract** (for extension implementers):
|
|
194
|
+
|
|
195
|
+
```ts
|
|
196
|
+
// bridge method: 'connector.probe'
|
|
197
|
+
// params:
|
|
198
|
+
{
|
|
199
|
+
pluginId: string;
|
|
200
|
+
host_match: string; // expanded with {settings.*}
|
|
201
|
+
login_redirect?: string; // expanded with {settings.*}
|
|
202
|
+
runner: 'main' | 'isolated'; // inherits from manifest.runner
|
|
203
|
+
timeout_ms: number; // honour or cap, your call
|
|
204
|
+
}
|
|
205
|
+
// response:
|
|
206
|
+
{ ok: true, url: '<final tab URL>' }
|
|
207
|
+
{ ok: false, error: 'login required' | '<other>', url?: string }
|
|
208
|
+
```
|
|
209
|
+
|
|
210
|
+
The extension's existing tab-acquisition logic already knows how
|
|
211
|
+
to handle `host_match` + `login_redirect`; the probe just runs the
|
|
212
|
+
acquire step and skips `executeScript`. If the extension isn't
|
|
213
|
+
connected, the Forge route surfaces a clear "install + sign in to
|
|
214
|
+
the Forge extension" message.
|
|
215
|
+
|
|
216
|
+
### shell protocol
|
|
217
|
+
|
|
218
|
+
For local CLI tools. **Every arg is templated independently** — no
|
|
219
|
+
shell injection.
|
|
220
|
+
|
|
221
|
+
```yaml
|
|
222
|
+
tools:
|
|
223
|
+
git_log:
|
|
224
|
+
protocol: shell
|
|
225
|
+
parameters:
|
|
226
|
+
repo: { type: string, required: true }
|
|
227
|
+
n: { type: number, default: 20 }
|
|
228
|
+
command: ['git', '-C', '{args.repo}', 'log', '-n', '{args.n}', '--oneline']
|
|
229
|
+
timeout_ms: 5000
|
|
230
|
+
```
|
|
231
|
+
|
|
232
|
+
## How you, the AI, install it
|
|
233
|
+
|
|
234
|
+
You have direct filesystem access to the running user's Forge data
|
|
235
|
+
directory. There are two paths:
|
|
236
|
+
|
|
237
|
+
### Path A — write the manifest directly (preferred for AI)
|
|
238
|
+
|
|
239
|
+
Find the data directory (usually `~/.forge/data/`, override via
|
|
240
|
+
`FORGE_DATA_DIR` env). Write:
|
|
241
|
+
|
|
242
|
+
```
|
|
243
|
+
<dataDir>/connectors/<id>/manifest.yaml
|
|
244
|
+
```
|
|
245
|
+
|
|
246
|
+
…then call the install-local API to register it:
|
|
247
|
+
|
|
248
|
+
```bash
|
|
249
|
+
curl -s -X POST http://localhost:8403/api/connectors/install-local \
|
|
250
|
+
-H "X-Forge-Token: $TOKEN" \
|
|
251
|
+
-H "Content-Type: application/json" \
|
|
252
|
+
-d "$(jq -nc --arg yaml "$(cat <dataDir>/connectors/<id>/manifest.yaml)" '{yaml: $yaml}')"
|
|
253
|
+
```
|
|
254
|
+
|
|
255
|
+
(See `lib/help-docs/CLAUDE.md` for the auth pattern.)
|
|
256
|
+
|
|
257
|
+
### Path B — generate a zip for hand-installation
|
|
258
|
+
|
|
259
|
+
If the user wants to share the connector with a colleague or store
|
|
260
|
+
it in version control, package it as a zip:
|
|
261
|
+
|
|
262
|
+
```
|
|
263
|
+
my-connector.zip
|
|
264
|
+
├── manifest.yaml
|
|
265
|
+
├── README.md (optional)
|
|
266
|
+
├── icon.svg (optional)
|
|
267
|
+
└── tools/ (optional — separate files when scripts grow long)
|
|
268
|
+
└── list_my_issues.js
|
|
269
|
+
```
|
|
270
|
+
|
|
271
|
+
`manifest.yaml` must be at the root. The user then uploads via the
|
|
272
|
+
"+ Upload" button in Settings → Connectors (or drags-and-drops onto
|
|
273
|
+
the panel).
|
|
274
|
+
|
|
275
|
+
## Checklist before declaring done
|
|
276
|
+
|
|
277
|
+
- [ ] `id` is lowercase + URL-safe
|
|
278
|
+
- [ ] At least one tool defined
|
|
279
|
+
- [ ] `description` reads like one a stranger would understand
|
|
280
|
+
- [ ] `version: "0.1.0"` (or whatever the user specifies)
|
|
281
|
+
- [ ] `host_match` + `login_redirect` set for browser connectors
|
|
282
|
+
- [ ] Settings include any secrets as `type: secret` (encrypted at rest)
|
|
283
|
+
- [ ] Run a 1-tool smoke test by triggering the chat agent to call it,
|
|
284
|
+
and surface any error to the user
|
|
285
|
+
- [ ] Tell the user where the manifest was saved + that they can edit
|
|
286
|
+
it under `<dataDir>/connectors/<id>/manifest.yaml`
|
|
287
|
+
|
|
288
|
+
## Iterating
|
|
289
|
+
|
|
290
|
+
When the user reports a bug ("the list_my_issues tool returned 0 rows"):
|
|
291
|
+
|
|
292
|
+
1. Open the page they were on (they can navigate, you can suggest URLs).
|
|
293
|
+
2. Use the browser extension's DOM inspector (or `mcp__chrome__` if
|
|
294
|
+
available) to find a stable selector.
|
|
295
|
+
3. Edit the script body in the manifest. Bump `version` (patch bump).
|
|
296
|
+
4. Tell the user to either close + reopen the Settings panel
|
|
297
|
+
("Refresh" in Connectors), or just re-invoke the tool — the
|
|
298
|
+
extension picks up the new manifest on next call.
|
|
299
|
+
|
|
300
|
+
## Limits and gotchas
|
|
301
|
+
|
|
302
|
+
- Browser scripts cannot use Chrome extension APIs (`chrome.*`) — only
|
|
303
|
+
page-context globals.
|
|
304
|
+
- For sites that virtualise lists (react-window, ag-grid), scroll
|
|
305
|
+
programmatically inside the script before scraping; otherwise you
|
|
306
|
+
only see what's currently in the DOM.
|
|
307
|
+
- For strict-CSP sites (Teams, github.com), pure `eval` is blocked.
|
|
308
|
+
Use `runner: isolated` (which the extension's MV3 sandbox enables)
|
|
309
|
+
but you lose access to `window` globals (MSAL tokens, etc.) — stick
|
|
310
|
+
to pure DOM extraction.
|
|
311
|
+
- If a connector breaks after a site redesign, only the YAML's
|
|
312
|
+
`script` body needs to change. Bump version, save, retry — the
|
|
313
|
+
registry-based update path is for connectors that came from a
|
|
314
|
+
shared `forge-connectors` repo.
|
package/lib/help-docs/CLAUDE.md
CHANGED
|
@@ -46,7 +46,8 @@ The token is valid for 24 hours. Store it in a variable and reuse for all API ca
|
|
|
46
46
|
| `13-ide-plugins.md` | VSCode extension + IntelliJ plugin — install, tabs, multi-connection, agent terminal launching |
|
|
47
47
|
| `15-crafts.md` | Crafts — project-scoped mini-app tabs with SDK; AI-generated via "+ Craft" button |
|
|
48
48
|
| `16-gitlab-autofix.md` | GitLab issue auto-fix — worktree + base-branch rule + Premium epic context + image attachments |
|
|
49
|
-
| `17-connectors.md` |
|
|
49
|
+
| `17-connectors.md` | Connectors — independent subsystem, marketplace fetched from `forge-connectors` registry, no built-ins. Manifest schema, HTTP API, install/update flow, pre-v0.9 migration. |
|
|
50
|
+
| `21-build-connector.md` | **Authoring** a custom connector — interview script, manifest template (browser / http / shell protocols), how to install locally via the Forge data dir or a zip upload. Use this when the user asks to BUILD a connector, not when they ask about an existing one. |
|
|
50
51
|
| `18-chrome-mcp.md` | Connect Forge Claude Code sessions to a real Chrome via chrome-devtools-mcp — dev-time browser access for connector authoring |
|
|
51
52
|
| `19-jobs.md` | Jobs — scheduled connector polls that dedup and fan out to Pipeline / Chat |
|
|
52
53
|
| `20-mantis-bug-fix.md` | Mantis → Bug Fix → MR builtin pipeline (mantis-bug-fix-and-mr) |
|
|
@@ -75,7 +76,8 @@ The token is valid for 24 hours. Store it in a variable and reuse for all API ca
|
|
|
75
76
|
- vsce/vsix/JetBrains marketplace publish → `13-ide-plugins.md`
|
|
76
77
|
- Craft/custom tab/mini-app/extend project/AI-generated tab/builder → `15-crafts.md`
|
|
77
78
|
- GitLab/glab/MR/merge request/issue auto-fix/epic → `16-gitlab-autofix.md`
|
|
78
|
-
- Connector/
|
|
79
|
+
- Connector/connector marketplace/install connector/forge-connectors registry/Mantis manifest → `17-connectors.md`
|
|
80
|
+
- Build / author / write / create a new connector / "make me a connector for X" / custom connector → `21-build-connector.md`
|
|
79
81
|
- Chrome MCP / chrome-devtools-mcp / dev-time browser / CDP / remote debugging → `18-chrome-mcp.md`
|
|
80
82
|
- Job / scheduled job / connector poll / dedup / periodic fetch / Teams poll / Mantis bug poll → `19-jobs.md`
|
|
81
83
|
- Mantis bug fix pipeline / mantis-bug-fix-and-mr / open MR for Mantis bug / notify Teams from pipeline / connector-tool endpoint → `20-mantis-bug-fix.md`
|
package/lib/init.ts
CHANGED
|
@@ -133,6 +133,31 @@ export function ensureInitialized() {
|
|
|
133
133
|
setInterval(() => { syncSkills().catch(() => {}); }, 60 * 60 * 1000);
|
|
134
134
|
} catch {}
|
|
135
135
|
|
|
136
|
+
// One-shot migration: move pre-existing connector rows out of the
|
|
137
|
+
// plugin registry and into <dataDir>/connectors/. Idempotent — safe
|
|
138
|
+
// to call on every boot. Must run BEFORE syncRegistry so the
|
|
139
|
+
// migrated rows are visible to the sync layer (which uses
|
|
140
|
+
// installed_version to decide what to refresh).
|
|
141
|
+
try {
|
|
142
|
+
time('migrateConnectorConfigs', () => {
|
|
143
|
+
const { migrateConnectorConfigs } = require('./connectors/migration');
|
|
144
|
+
migrateConnectorConfigs();
|
|
145
|
+
});
|
|
146
|
+
} catch (err) {
|
|
147
|
+
console.warn('[connectors] migration failed:', err);
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
// Sync connectors registry (async, non-blocking). Refresh installed
|
|
151
|
+
// manifests in case the user has older versions on disk. Same cadence
|
|
152
|
+
// as skills.
|
|
153
|
+
try {
|
|
154
|
+
const { syncRegistry } = require('./connectors/sync');
|
|
155
|
+
syncRegistry({ refreshInstalled: true }).catch(() => {});
|
|
156
|
+
setInterval(() => {
|
|
157
|
+
syncRegistry({ refreshInstalled: true }).catch(() => {});
|
|
158
|
+
}, 60 * 60 * 1000);
|
|
159
|
+
} catch {}
|
|
160
|
+
|
|
136
161
|
// Usage scanner — defer to next tick so it doesn't block ensureInitialized().
|
|
137
162
|
// On a host with hundreds of project dirs in ~/.claude/projects/, the
|
|
138
163
|
// synchronous readdirSync + statSync loop can take 5-10s; running it on
|
package/lib/jobs/dispatcher.ts
CHANGED
|
@@ -58,7 +58,12 @@ function renderTemplateMap(map: Record<string, string>, item: unknown): Record<s
|
|
|
58
58
|
* slot). Letting pipeline_runs.dedup_key be NULL means each dispatch
|
|
59
59
|
* gets its own row.
|
|
60
60
|
*/
|
|
61
|
-
export function dispatchToPipeline(
|
|
61
|
+
export function dispatchToPipeline(
|
|
62
|
+
params: PipelineDispatchParams,
|
|
63
|
+
item: unknown,
|
|
64
|
+
_dedupKey: string,
|
|
65
|
+
opts: { skills?: string[] } = {},
|
|
66
|
+
): { target_id: string; rendered_input: Record<string, string>; empty_keys: string[] } {
|
|
62
67
|
const renderedInput = renderTemplateMap(params.input_template || {}, item);
|
|
63
68
|
// Heads-up if a template key rendered to empty — almost always means the
|
|
64
69
|
// user wrote {{item.foo}} but the item has no 'foo' field. The pipeline
|
|
@@ -80,7 +85,8 @@ export function dispatchToPipeline(params: PipelineDispatchParams, item: unknown
|
|
|
80
85
|
params.project_name,
|
|
81
86
|
params.workflow_name,
|
|
82
87
|
renderedInput,
|
|
83
|
-
// intentionally no dedup_key — see comment above
|
|
88
|
+
/* dedupKey */ undefined, // intentionally no dedup_key — see comment above
|
|
89
|
+
{ skills: opts.skills },
|
|
84
90
|
);
|
|
85
91
|
return { target_id: result.pipelineId, rendered_input: renderedInput, empty_keys: emptyKeys };
|
|
86
92
|
}
|
|
@@ -114,7 +120,12 @@ async function getMainSessionId(): Promise<string> {
|
|
|
114
120
|
*/
|
|
115
121
|
const reusedSessionByJob = new Map<string, string>();
|
|
116
122
|
|
|
117
|
-
export async function dispatchToChat(
|
|
123
|
+
export async function dispatchToChat(
|
|
124
|
+
params: ChatDispatchParams,
|
|
125
|
+
item: unknown,
|
|
126
|
+
jobId: string,
|
|
127
|
+
opts: { skills?: string[] } = {},
|
|
128
|
+
): Promise<{ target_id: string }> {
|
|
118
129
|
let sessionId: string | undefined;
|
|
119
130
|
if (params.target === 'main') {
|
|
120
131
|
sessionId = await getMainSessionId();
|
|
@@ -136,9 +147,15 @@ export async function dispatchToChat(params: ChatDispatchParams, item: unknown,
|
|
|
136
147
|
if (params.reuse_session) reusedSessionByJob.set(jobId, sessionId);
|
|
137
148
|
}
|
|
138
149
|
|
|
139
|
-
const
|
|
140
|
-
if (!
|
|
141
|
-
|
|
150
|
+
const rendered = renderTemplate(params.message_template, item);
|
|
151
|
+
if (!rendered.trim()) throw new Error('rendered chat message is empty');
|
|
152
|
+
// Skills get prepended as a one-line directive in the chat message —
|
|
153
|
+
// the chat agent picks them up the same way the user would by typing
|
|
154
|
+
// /skill-name themselves.
|
|
155
|
+
const skillsPrefix = opts.skills && opts.skills.length
|
|
156
|
+
? `Use these Forge skills as appropriate for this task: ${opts.skills.map(s => '/' + s).join(', ')}.\n\n`
|
|
157
|
+
: '';
|
|
158
|
+
await chatHttp(`/api/sessions/${encodeURIComponent(sessionId)}/messages`, { text: skillsPrefix + rendered });
|
|
142
159
|
return { target_id: sessionId };
|
|
143
160
|
}
|
|
144
161
|
|
|
@@ -158,7 +175,7 @@ export async function dispatchToChatSummary(
|
|
|
158
175
|
params: ChatDispatchParams,
|
|
159
176
|
items: unknown[],
|
|
160
177
|
jobId: string,
|
|
161
|
-
opts: { totalMatching?: number | string | null } = {},
|
|
178
|
+
opts: { totalMatching?: number | string | null; skills?: string[] } = {},
|
|
162
179
|
): Promise<{ target_id: string; count: number }> {
|
|
163
180
|
let sessionId: string | undefined;
|
|
164
181
|
// Default target for summary mode is the main chat — that's where the
|
|
@@ -208,7 +225,10 @@ export async function dispatchToChatSummary(
|
|
|
208
225
|
text = renderTemplate(text, items[0] ?? {});
|
|
209
226
|
|
|
210
227
|
if (!text.trim()) throw new Error('rendered chat summary message is empty');
|
|
211
|
-
|
|
228
|
+
const skillsPrefix = opts.skills && opts.skills.length
|
|
229
|
+
? `Use these Forge skills as appropriate for this task: ${opts.skills.map(s => '/' + s).join(', ')}.\n\n`
|
|
230
|
+
: '';
|
|
231
|
+
await chatHttp(`/api/sessions/${encodeURIComponent(sessionId)}/messages`, { text: skillsPrefix + text });
|
|
212
232
|
return { target_id: sessionId, count: items.length };
|
|
213
233
|
}
|
|
214
234
|
|
package/lib/jobs/scheduler.ts
CHANGED
|
@@ -50,11 +50,53 @@ async function tick(): Promise<void> {
|
|
|
50
50
|
}
|
|
51
51
|
}
|
|
52
52
|
|
|
53
|
+
function toSqlIso(d: Date): string {
|
|
54
|
+
return d.toISOString().replace('T', ' ').slice(0, 19);
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
/**
|
|
58
|
+
* Compute and store the job's next_run_at based on schedule_kind.
|
|
59
|
+
* For 'once' jobs we also auto-disable after the firing tick so they
|
|
60
|
+
* don't fire repeatedly when their schedule_at time is in the past.
|
|
61
|
+
*/
|
|
53
62
|
function advanceSchedule(job: Job): void {
|
|
54
|
-
const next = new Date(Date.now() + Math.max(1, job.schedule_interval_minutes) * 60_000);
|
|
55
|
-
// setNextRunAt also bumps last_run_at — call updateJob via raw SQL for clarity.
|
|
56
63
|
const { setNextRunAt } = require('./store') as typeof import('./store');
|
|
57
|
-
|
|
64
|
+
const now = Date.now();
|
|
65
|
+
|
|
66
|
+
if (job.schedule_kind === 'manual') {
|
|
67
|
+
// Manual jobs are filtered out of getDueJobs and should never get
|
|
68
|
+
// here. Belt-and-suspenders: clear any stray next_run_at.
|
|
69
|
+
setNextRunAt(job.id, null);
|
|
70
|
+
return;
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
if (job.schedule_kind === 'once') {
|
|
74
|
+
// One-shot: this tick is the fire. Disable for future, no next run.
|
|
75
|
+
setNextRunAt(job.id, null);
|
|
76
|
+
try { updateJob(job.id, { enabled: false }); } catch (e) {
|
|
77
|
+
console.warn(`[jobs] failed to auto-disable one-shot ${job.id}:`, e);
|
|
78
|
+
}
|
|
79
|
+
return;
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
if (job.schedule_kind === 'cron' && job.schedule_cron) {
|
|
83
|
+
try {
|
|
84
|
+
const { CronExpressionParser } = require('cron-parser');
|
|
85
|
+
const iter = CronExpressionParser.parse(job.schedule_cron, { currentDate: new Date(now) });
|
|
86
|
+
const next = iter.next().toDate();
|
|
87
|
+
setNextRunAt(job.id, toSqlIso(next));
|
|
88
|
+
return;
|
|
89
|
+
} catch (e) {
|
|
90
|
+
console.warn(`[jobs] cron parse failed for ${job.id} (expr "${job.schedule_cron}"):`, (e as Error).message);
|
|
91
|
+
// Fall through to period-style backoff so a broken cron doesn't
|
|
92
|
+
// tight-loop the tick.
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
// Default / 'period' path.
|
|
97
|
+
const minutes = Math.max(1, job.schedule_interval_minutes || 30);
|
|
98
|
+
const next = new Date(now + minutes * 60_000);
|
|
99
|
+
setNextRunAt(job.id, toSqlIso(next));
|
|
58
100
|
}
|
|
59
101
|
|
|
60
102
|
/**
|
|
@@ -220,7 +262,7 @@ export async function executeRun(job: Job, runId: string): Promise<void> {
|
|
|
220
262
|
? (parsed as any).total_matching ?? (parsed as any).total ?? null
|
|
221
263
|
: null;
|
|
222
264
|
try {
|
|
223
|
-
const out = await dispatchToChatSummary(chatParams, newItems, job.id, { totalMatching });
|
|
265
|
+
const out = await dispatchToChatSummary(chatParams, newItems, job.id, { totalMatching, skills: job.skills });
|
|
224
266
|
recordDispatch({
|
|
225
267
|
job_run_id: runId,
|
|
226
268
|
item_key: `summary-${runId}`,
|
|
@@ -262,10 +304,28 @@ export async function executeRun(job: Job, runId: string): Promise<void> {
|
|
|
262
304
|
const preview = renderItemPreview(item);
|
|
263
305
|
logLine('info', `[${idx}] ${key} — new — dispatching ${job.dispatch_type}…`);
|
|
264
306
|
const dispatchStart = Date.now();
|
|
307
|
+
// Auto-install any skill named on the job into the dispatch
|
|
308
|
+
// target project, so the spawned task can actually invoke them.
|
|
309
|
+
// Skipped for chat dispatches — chat backend already sees globally-
|
|
310
|
+
// installed skills and we don't have a target project here.
|
|
311
|
+
if (job.skills && job.skills.length && job.dispatch_type === 'pipeline') {
|
|
312
|
+
const targetProject = (job.dispatch_params as PipelineDispatchParams).project_path;
|
|
313
|
+
if (targetProject) {
|
|
314
|
+
for (const skillName of job.skills) {
|
|
315
|
+
try {
|
|
316
|
+
const { ensureInstalledInProject } = require('../skills');
|
|
317
|
+
const r = await ensureInstalledInProject(skillName, targetProject);
|
|
318
|
+
if (!r.installed) logLine('warn', `skill "${skillName}" not installable: ${r.reason}`);
|
|
319
|
+
} catch (err) {
|
|
320
|
+
logLine('warn', `skill "${skillName}" install failed: ${(err as Error).message}`);
|
|
321
|
+
}
|
|
322
|
+
}
|
|
323
|
+
}
|
|
324
|
+
}
|
|
265
325
|
try {
|
|
266
326
|
const out = job.dispatch_type === 'pipeline'
|
|
267
|
-
? dispatchToPipeline(job.dispatch_params as PipelineDispatchParams, item, key)
|
|
268
|
-
: await dispatchToChat(job.dispatch_params as ChatDispatchParams, item, job.id);
|
|
327
|
+
? dispatchToPipeline(job.dispatch_params as PipelineDispatchParams, item, key, { skills: job.skills })
|
|
328
|
+
: await dispatchToChat(job.dispatch_params as ChatDispatchParams, item, job.id, { skills: job.skills });
|
|
269
329
|
recordDispatch({
|
|
270
330
|
job_run_id: runId, item_key: key, item_preview: preview,
|
|
271
331
|
dispatch_type: job.dispatch_type, dispatch_target_id: out.target_id, status: 'dispatched',
|
package/lib/jobs/store.ts
CHANGED
|
@@ -31,6 +31,15 @@ export function ensureSchema(): void {
|
|
|
31
31
|
dedup_field TEXT NOT NULL,
|
|
32
32
|
dispatch_type TEXT NOT NULL,
|
|
33
33
|
dispatch_params TEXT NOT NULL DEFAULT '{}',
|
|
34
|
+
/** JSON array of skill names (from forge-skills registry).
|
|
35
|
+
Composed into the task system prompt at dispatch time. */
|
|
36
|
+
skills TEXT NOT NULL DEFAULT '[]',
|
|
37
|
+
/** 'period' (default — fire every schedule_interval_minutes),
|
|
38
|
+
'once' (fire once at schedule_at, then auto-disable), or
|
|
39
|
+
'cron' (fire on each cron tick per schedule_cron). */
|
|
40
|
+
schedule_kind TEXT NOT NULL DEFAULT 'period',
|
|
41
|
+
schedule_at TEXT,
|
|
42
|
+
schedule_cron TEXT,
|
|
34
43
|
last_run_at TEXT,
|
|
35
44
|
next_run_at TEXT,
|
|
36
45
|
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
|
@@ -74,6 +83,11 @@ export function ensureSchema(): void {
|
|
|
74
83
|
// Migrations for already-existing job_runs tables.
|
|
75
84
|
try { db().exec(`ALTER TABLE job_runs ADD COLUMN notes TEXT`); } catch {}
|
|
76
85
|
try { db().exec(`ALTER TABLE job_runs ADD COLUMN log TEXT`); } catch {}
|
|
86
|
+
// Migration for already-existing jobs table.
|
|
87
|
+
try { db().exec(`ALTER TABLE jobs ADD COLUMN skills TEXT NOT NULL DEFAULT '[]'`); } catch {}
|
|
88
|
+
try { db().exec(`ALTER TABLE jobs ADD COLUMN schedule_kind TEXT NOT NULL DEFAULT 'period'`); } catch {}
|
|
89
|
+
try { db().exec(`ALTER TABLE jobs ADD COLUMN schedule_at TEXT`); } catch {}
|
|
90
|
+
try { db().exec(`ALTER TABLE jobs ADD COLUMN schedule_cron TEXT`); } catch {}
|
|
77
91
|
ensured = true;
|
|
78
92
|
}
|
|
79
93
|
|
|
@@ -92,6 +106,10 @@ function rowToJob(r: any): Job {
|
|
|
92
106
|
dedup_field: r.dedup_field,
|
|
93
107
|
dispatch_type: r.dispatch_type,
|
|
94
108
|
dispatch_params: safeParse(r.dispatch_params, {}) as DispatchParams,
|
|
109
|
+
skills: safeParse(r.skills, []) as string[],
|
|
110
|
+
schedule_kind: (r.schedule_kind as 'period' | 'once' | 'cron' | 'manual') || 'period',
|
|
111
|
+
schedule_at: toIsoUTC(r.schedule_at),
|
|
112
|
+
schedule_cron: r.schedule_cron || null,
|
|
95
113
|
last_run_at: toIsoUTC(r.last_run_at),
|
|
96
114
|
next_run_at: toIsoUTC(r.next_run_at),
|
|
97
115
|
created_at: toIsoUTC(r.created_at) || r.created_at,
|
|
@@ -156,8 +174,9 @@ export function createJob(input: CreateJobInput): Job {
|
|
|
156
174
|
INSERT INTO jobs (id, name, enabled, schedule_interval_minutes,
|
|
157
175
|
source_connector, source_tool, source_input,
|
|
158
176
|
items_path, dedup_field,
|
|
159
|
-
dispatch_type, dispatch_params
|
|
160
|
-
|
|
177
|
+
dispatch_type, dispatch_params, skills,
|
|
178
|
+
schedule_kind, schedule_at, schedule_cron)
|
|
179
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
161
180
|
`).run(
|
|
162
181
|
id,
|
|
163
182
|
input.name,
|
|
@@ -170,6 +189,10 @@ export function createJob(input: CreateJobInput): Job {
|
|
|
170
189
|
input.dedup_field,
|
|
171
190
|
input.dispatch_type,
|
|
172
191
|
JSON.stringify(input.dispatch_params),
|
|
192
|
+
JSON.stringify(Array.isArray(input.skills) ? input.skills : []),
|
|
193
|
+
input.schedule_kind || 'period',
|
|
194
|
+
input.schedule_at || null,
|
|
195
|
+
input.schedule_cron || null,
|
|
173
196
|
);
|
|
174
197
|
|
|
175
198
|
// Backfill guard: if mark_existing_as_seen is true (default), we don't pre-seed
|
|
@@ -182,6 +205,21 @@ export function createJob(input: CreateJobInput): Job {
|
|
|
182
205
|
db().prepare('UPDATE jobs SET source_input = ? WHERE id = ?').run(JSON.stringify(inputWithFlag), id);
|
|
183
206
|
}
|
|
184
207
|
|
|
208
|
+
// Seed next_run_at so the scheduler picks the job up at the right
|
|
209
|
+
// moment instead of immediately (which getDueJobs would otherwise
|
|
210
|
+
// do for any NULL next_run_at).
|
|
211
|
+
if (input.schedule_kind === 'once' && input.schedule_at) {
|
|
212
|
+
const t = new Date(input.schedule_at);
|
|
213
|
+
if (!Number.isNaN(t.getTime())) setNextRunAt(id, t.toISOString().replace('T', ' ').slice(0, 19));
|
|
214
|
+
} else if (input.schedule_kind === 'cron' && input.schedule_cron) {
|
|
215
|
+
try {
|
|
216
|
+
const { CronExpressionParser } = require('cron-parser');
|
|
217
|
+
const iter = CronExpressionParser.parse(input.schedule_cron, { currentDate: new Date() });
|
|
218
|
+
const next = iter.next().toDate();
|
|
219
|
+
setNextRunAt(id, next.toISOString().replace('T', ' ').slice(0, 19));
|
|
220
|
+
} catch {}
|
|
221
|
+
}
|
|
222
|
+
|
|
185
223
|
return getJob(id)!;
|
|
186
224
|
}
|
|
187
225
|
|
|
@@ -190,6 +228,10 @@ export function updateJob(id: string, patch: Partial<{
|
|
|
190
228
|
source_connector: string; source_tool: string; source_input: Record<string, unknown>;
|
|
191
229
|
items_path: string; dedup_field: string;
|
|
192
230
|
dispatch_type: 'pipeline' | 'chat'; dispatch_params: DispatchParams;
|
|
231
|
+
skills: string[];
|
|
232
|
+
schedule_kind: 'period' | 'once' | 'cron' | 'manual';
|
|
233
|
+
schedule_at: string | null;
|
|
234
|
+
schedule_cron: string | null;
|
|
193
235
|
}>): boolean {
|
|
194
236
|
ensureSchema();
|
|
195
237
|
const sets: string[] = []; const vals: any[] = [];
|
|
@@ -203,6 +245,10 @@ export function updateJob(id: string, patch: Partial<{
|
|
|
203
245
|
if (patch.dedup_field !== undefined) { sets.push('dedup_field = ?'); vals.push(patch.dedup_field); }
|
|
204
246
|
if (patch.dispatch_type !== undefined) { sets.push('dispatch_type = ?'); vals.push(patch.dispatch_type); }
|
|
205
247
|
if (patch.dispatch_params !== undefined) { sets.push('dispatch_params = ?'); vals.push(JSON.stringify(patch.dispatch_params)); }
|
|
248
|
+
if (patch.skills !== undefined) { sets.push('skills = ?'); vals.push(JSON.stringify(Array.isArray(patch.skills) ? patch.skills : [])); }
|
|
249
|
+
if (patch.schedule_kind !== undefined) { sets.push('schedule_kind = ?'); vals.push(patch.schedule_kind); }
|
|
250
|
+
if (patch.schedule_at !== undefined) { sets.push('schedule_at = ?'); vals.push(patch.schedule_at); }
|
|
251
|
+
if (patch.schedule_cron !== undefined) { sets.push('schedule_cron = ?'); vals.push(patch.schedule_cron); }
|
|
206
252
|
if (sets.length === 0) return false;
|
|
207
253
|
sets.push("updated_at = datetime('now')");
|
|
208
254
|
vals.push(id);
|
|
@@ -224,9 +270,12 @@ export function setNextRunAt(id: string, nextRunAt: string | null): void {
|
|
|
224
270
|
/** Jobs due to run: enabled AND (next_run_at IS NULL OR next_run_at <= now). */
|
|
225
271
|
export function getDueJobs(): Job[] {
|
|
226
272
|
ensureSchema();
|
|
273
|
+
// Exclude schedule_kind='manual' — those only run when explicitly
|
|
274
|
+
// fired via /api/jobs/[id]/fire, never on the scheduler tick.
|
|
227
275
|
const rows = db().prepare(`
|
|
228
276
|
SELECT * FROM jobs
|
|
229
277
|
WHERE enabled = 1
|
|
278
|
+
AND schedule_kind != 'manual'
|
|
230
279
|
AND (next_run_at IS NULL OR next_run_at <= datetime('now'))
|
|
231
280
|
ORDER BY (next_run_at IS NULL) DESC, next_run_at ASC
|
|
232
281
|
`).all() as any[];
|