@audiofool/moondog 0.3.0-beta.10 → 0.3.0-beta.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +34 -9
- package/package.json +2 -2
- package/scripts/moondog.mjs +10 -4
- package/src/core/apple-projection-domain-services.mjs +4 -0
- package/src/core/capability-catalog.mjs +11 -0
- package/src/core/library-store.mjs +48 -4
- package/src/core/listening-profile-domain-services.mjs +4 -0
- package/src/core/moondog-application.mjs +105 -4
- package/src/i18n/screen/es.mjs +18 -4
- package/src/i18n/screen/ja.mjs +18 -4
- package/src/i18n/screen/pt.mjs +18 -4
- package/src/i18n/screen/zh.mjs +18 -4
- package/src/integrations/playlist-links.mjs +27 -9
- package/src/integrations/public-playlists.mjs +32 -5
- package/src/integrations/spotify/service.mjs +6 -1
- package/src/integrations/spotify/web-api-client.mjs +8 -2
- package/src/integrations/web/deepseek-web.mjs +1 -1
- package/src/integrations/web/web-research.mjs +84 -17
- package/src/integrations/web/web-result.mjs +60 -12
- package/src/profile/listening-history-store.mjs +54 -0
- package/src/runtime/pi/agent-runtime.mjs +73 -11
- package/src/runtime/pi/configured-runtime.mjs +3 -0
- package/src/runtime/pi/image-input.mjs +148 -0
- package/src/runtime/pi/model-roles.mjs +121 -0
- package/src/runtime/pi/model-settings.mjs +63 -2
- package/src/runtime/pi/runtime-settings.mjs +4 -2
- package/src/surfaces/cli/brand-components.mjs +1 -1
- package/src/surfaces/cli/command-completions.mjs +42 -3
- package/src/surfaces/cli/format-output.mjs +1 -1
- package/src/surfaces/cli/model-roles-command.mjs +60 -0
- package/src/surfaces/cli/tui.mjs +17 -3
|
@@ -54,6 +54,30 @@ async function boundedJson(response) {
|
|
|
54
54
|
catch { throw new Error("The service changed its playlist response or requires sign-in. No music was imported."); }
|
|
55
55
|
}
|
|
56
56
|
|
|
57
|
+
// A NetEase playlist, as { code, result: { id, name, privacy, trackCount, tracks } }. NetEase's web player reads
|
|
58
|
+
// playlists with v6, which answers every public one, a listener's Liked songs among them, with all its song ids but
|
|
59
|
+
// details for only the first few without a sign-in; the rest come from song/detail, 200 at a time, in the
|
|
60
|
+
// playlist's order. The older endpoint, which some playlists answer with code 20001, is the fallback.
|
|
61
|
+
const NETEASE_SONGS_MAX = 3000;
|
|
62
|
+
async function neteasePlaylist(id, fetchImpl) {
|
|
63
|
+
const referer = "https://music.163.com/";
|
|
64
|
+
const current = await boundedJson(await responseFor(`https://music.163.com/api/v6/playlist/detail?id=${id}&n=100000`, fetchImpl, referer)).catch(() => null);
|
|
65
|
+
const playlist = current?.code === 200 ? current.playlist : null;
|
|
66
|
+
if (!playlist || !Array.isArray(playlist.trackIds)) {
|
|
67
|
+
return boundedJson(await responseFor(`https://music.163.com/api/playlist/detail?id=${id}`, fetchImpl, referer));
|
|
68
|
+
}
|
|
69
|
+
const ids = playlist.trackIds.map((entry) => String(entry?.id ?? "")).filter((value) => numericId.test(value)).slice(0, NETEASE_SONGS_MAX);
|
|
70
|
+
const songs = new Map((Array.isArray(playlist.tracks) ? playlist.tracks : []).map((song) => [String(song?.id ?? ""), song]));
|
|
71
|
+
const missing = ids.filter((songId) => !songs.has(songId));
|
|
72
|
+
for (let start = 0; start < missing.length; start += 200) {
|
|
73
|
+
const batch = missing.slice(start, start + 200);
|
|
74
|
+
const detail = await boundedJson(await responseFor(`https://music.163.com/api/song/detail/?ids=${encodeURIComponent(`[${batch.join(",")}]`)}`, fetchImpl, referer));
|
|
75
|
+
for (const song of Array.isArray(detail?.songs) ? detail.songs : []) songs.set(String(song?.id ?? ""), song);
|
|
76
|
+
}
|
|
77
|
+
return { code: 200, result: { id: playlist.id, name: playlist.name, coverImgUrl: playlist.coverImgUrl, privacy: playlist.privacy, trackCount: playlist.trackCount ?? ids.length,
|
|
78
|
+
tracks: ids.map((songId) => songs.get(songId)).filter(Boolean) } };
|
|
79
|
+
}
|
|
80
|
+
|
|
57
81
|
/** A public QQ Music or NetEase playlist by its share link: { provider, id, url, name, total, rows, skipped }. */
|
|
58
82
|
export async function fetchPublicPlaylist({ input, provider, fetchImpl = fetch }) {
|
|
59
83
|
let url = playlistLink(input);
|
|
@@ -70,10 +94,9 @@ export async function fetchPublicPlaylist({ input, provider, fetchImpl = fetch }
|
|
|
70
94
|
if (!identity) throw new Error("Open this share link in your browser, then copy the full playlist page URL. Song, album and private playlist links are not supported.");
|
|
71
95
|
if (provider && ["qq_music", "netease"].includes(provider) && provider !== identity.provider) throw new Error("This link belongs to another service. Go back and choose that service, or paste the matching playlist link.");
|
|
72
96
|
const qq = identity.provider === "qq_music";
|
|
73
|
-
const
|
|
74
|
-
? `https://c.y.qq.com/qzone/fcg-bin/fcg_ucc_getcdinfo_byids_cp.fcg?type=1&json=1&utf8=1&onlysong=0&disstid=${identity.id}&format=json
|
|
75
|
-
:
|
|
76
|
-
const data = await boundedJson(await responseFor(endpoint, fetchImpl, qq ? "https://y.qq.com/" : "https://music.163.com/"));
|
|
97
|
+
const data = qq
|
|
98
|
+
? await boundedJson(await responseFor(`https://c.y.qq.com/qzone/fcg-bin/fcg_ucc_getcdinfo_byids_cp.fcg?type=1&json=1&utf8=1&onlysong=0&disstid=${identity.id}&format=json`, fetchImpl, "https://y.qq.com/"))
|
|
99
|
+
: await neteasePlaylist(identity.id, fetchImpl);
|
|
77
100
|
const playlist = qq ? data?.cdlist?.[0] : data?.result;
|
|
78
101
|
if ((qq ? data?.code !== 0 || data?.subcode !== 0 : data?.code !== 200) || !playlist
|
|
79
102
|
|| (qq ? Number(playlist.dir_show) === 0 : Number(playlist.privacy) === 10)) throw new Error("This playlist is unavailable publicly. Choose a playlist you can open without signing in; no account password or cookies are needed.");
|
|
@@ -91,10 +114,14 @@ export async function fetchPublicPlaylist({ input, provider, fetchImpl = fetch }
|
|
|
91
114
|
title: musicText(qq ? song.songname : song.name),
|
|
92
115
|
artist: musicText(Array.isArray(artists) ? artists.map((artist) => musicText(artist?.name)).filter(Boolean).join(", ") : ""),
|
|
93
116
|
album: musicText(qq ? song.albumname : (song.album ?? song.al)?.name),
|
|
117
|
+
// NetEase names each album cover on its own image host; it is shown over https.
|
|
118
|
+
...(!qq && /^https?:\/\/p[0-9]+\.music\.126\.net\//u.test((song.album ?? song.al)?.picUrl ?? "") ? { artwork: (song.album ?? song.al).picUrl.replace(/^http:/u, "https:") } : {}),
|
|
94
119
|
playlistId: identity.id, playlistName: name, position: index + 1, member: "public-playlist.json",
|
|
95
120
|
};
|
|
96
121
|
}).filter((row) => row.id && row.title);
|
|
97
|
-
|
|
122
|
+
// NetEase's own image host, shown over https, as a song's cover is.
|
|
123
|
+
const cover = !qq && /^https?:\/\/p[0-9]+\.music\.126\.net\//u.test(playlist.coverImgUrl ?? "") ? playlist.coverImgUrl.replace(/^http:/u, "https:") : null;
|
|
124
|
+
return { provider: identity.provider, id: identity.id, url: url.href, name, total, rows, skipped: songs.length - rows.length, ...(cover ? { cover } : {}) };
|
|
98
125
|
}
|
|
99
126
|
|
|
100
127
|
export async function readPublicPlaylist({ input, subjectId, capturedAt, provider, fetchImpl = fetch }) {
|
|
@@ -905,10 +905,15 @@ export function createSpotifyService(options = {}) {
|
|
|
905
905
|
if (!playlist) fail("spotify_playlist_unavailable", "Spotify did not describe this playlist.");
|
|
906
906
|
const page = await client.getPlaylistItems({ playlistId, limit: SPOTIFY_SERVICE_LIMITS.playlistItemsPageMax, offset: 0 }, { signal })
|
|
907
907
|
.catch((error) => (signal?.aborted ? Promise.reject(error) : null));
|
|
908
|
-
return { name: playlist.name, uri: playlist.uri, tracks_total: playlist.tracks_total ?? null,
|
|
908
|
+
return { name: playlist.name, uri: playlist.uri, tracks_total: playlist.tracks_total ?? null, ...(playlist.image ? { image: playlist.image } : {}),
|
|
909
909
|
tracks: page ? page.items.map((entry) => entry.item).filter(Boolean) : null };
|
|
910
910
|
},
|
|
911
911
|
|
|
912
|
+
// Only a playlist's cover, for one imported before Moondog kept it.
|
|
913
|
+
async readPlaylistCover({ playlistId } = {}, { signal } = {}) {
|
|
914
|
+
return (await client.getPlaylist({ playlistId: requiredPlaylistId(String(playlistId ?? "")) }, { signal }))?.image ?? null;
|
|
915
|
+
},
|
|
916
|
+
|
|
912
917
|
// A whole playlist for the listener's library: its name, whether the signed-in listener owns it, and its
|
|
913
918
|
// songs page by page, up to 2,000. Spotify's own editorial playlists answer most apps with not found.
|
|
914
919
|
async readPlaylist({ playlistId } = {}, { signal } = {}) {
|
|
@@ -160,14 +160,18 @@ function normalizeArtistNames(rawArtists) {
|
|
|
160
160
|
}
|
|
161
161
|
|
|
162
162
|
// Cover art a window can draw: one Spotify CDN image, the largest that is at most 640 pixels.
|
|
163
|
-
function coverImage(images) {
|
|
163
|
+
function coverImage(images, allowed = /^https:\/\/i\.scdn\.co\/image\/[A-Za-z0-9_-]{1,128}$/u) {
|
|
164
164
|
const usable = (Array.isArray(images) ? images : []).filter((image) =>
|
|
165
|
-
typeof image?.url === "string" &&
|
|
165
|
+
typeof image?.url === "string" && allowed.test(image.url));
|
|
166
166
|
const sized = usable.filter((image) => !Number.isInteger(image.width) || image.width <= 640)
|
|
167
167
|
.sort((left, right) => (right.width ?? 0) - (left.width ?? 0));
|
|
168
168
|
return (sized[0] ?? usable.at(-1))?.url;
|
|
169
169
|
}
|
|
170
170
|
|
|
171
|
+
// A playlist's cover lives on more hosts than an album's: one a listener uploaded on image-cdn-*.spotifycdn.com, the
|
|
172
|
+
// four-album mosaic on mosaic.scdn.co. The window's image policy (renderer/index.html) names the same two domains.
|
|
173
|
+
const PLAYLIST_COVER = /^https:\/\/[a-z0-9-]{1,40}\.(?:scdn\.co|spotifycdn\.com)\/[A-Za-z0-9_/-]{1,200}$/u;
|
|
174
|
+
|
|
171
175
|
// A track's album, kept so a song played alone can go on into the rest of its album.
|
|
172
176
|
// Hosts keep it next to the track URI and never show it to the model.
|
|
173
177
|
function albumUri(raw) {
|
|
@@ -462,6 +466,8 @@ function normalizePlaylist(payload, { includeControlMetadata = false } = {}) {
|
|
|
462
466
|
const snapshotId = safeText(payload.snapshot_id, 128);
|
|
463
467
|
if (ownerId) playlist.owner_id = ownerId;
|
|
464
468
|
if (snapshotId) playlist.snapshot_id = snapshotId;
|
|
469
|
+
const cover = coverImage(payload.images, PLAYLIST_COVER);
|
|
470
|
+
if (cover) playlist.image = cover;
|
|
465
471
|
}
|
|
466
472
|
const tracksTotal = safeInteger(
|
|
467
473
|
payload.items?.total ?? payload.tracks?.total,
|
|
@@ -69,7 +69,7 @@ export function createDeepSeekWebResearch({ apiKey, fetch: fetchImpl = globalThi
|
|
|
69
69
|
if (status !== 200) throw serviceError(status, body);
|
|
70
70
|
const title = typeof body.title === "string" && body.title.trim() ? clip(body.title.trim(), 300) : url;
|
|
71
71
|
return webResult({ provider: "deepseek", kind: "read", status: "found", retrievedAt,
|
|
72
|
-
summary: clip(text, webLimits.
|
|
72
|
+
summary: clip(text, webLimits.fullTextChars), truncated: text.length > webLimits.fullTextChars,
|
|
73
73
|
sources: [{ title, url, snippet: "", published_at: null }] });
|
|
74
74
|
}
|
|
75
75
|
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { readModelRoles } from "../../runtime/pi/model-roles.mjs";
|
|
1
2
|
import { createPersistentCredentialStore } from "../../runtime/pi/persistent-credential-store.mjs";
|
|
2
3
|
import { createBraveWebResearch } from "./brave-web.mjs";
|
|
3
4
|
import { createCodexWebResearch } from "./codex-web.mjs";
|
|
@@ -5,8 +6,13 @@ import { createDeepSeekWebResearch } from "./deepseek-web.mjs";
|
|
|
5
6
|
import { failure, webProviderName } from "./web-result.mjs";
|
|
6
7
|
|
|
7
8
|
// Search goes to the first backend that is set up; reading needs a backend that can open pages.
|
|
8
|
-
//
|
|
9
|
+
// The web search role in Settings (model-roles.json) picks one instead, and MOONDOG_WEB_SEARCH_PROVIDER
|
|
10
|
+
// and MOONDOG_WEB_READ_PROVIDER pick over that. As in Hermes Agent, a call that fails on the chosen
|
|
11
|
+
// backend is tried once more on each other backend that is set up, so one service being down or out
|
|
12
|
+
// of credit doesn't leave the agent without the web.
|
|
9
13
|
const order = Object.freeze({ search: ["deepseek", "brave", "codex"], read: ["deepseek", "codex"] });
|
|
14
|
+
// Failures that another backend would only repeat.
|
|
15
|
+
const final = new Set(["web_cancelled", "web_input_invalid", "web_url_invalid"]);
|
|
10
16
|
const providerIds = Object.freeze({ deepseek: "deepseek", brave: "brave", codex: "codex_cli" });
|
|
11
17
|
const keys = Object.freeze({
|
|
12
18
|
deepseek: { credential: "deepseek", variable: "DEEPSEEK_API_KEY" },
|
|
@@ -33,43 +39,103 @@ export async function createWebResearch({
|
|
|
33
39
|
credentials = createPersistentCredentialStore(environment),
|
|
34
40
|
fetch = globalThis.fetch,
|
|
35
41
|
createCodex = createCodexWebResearch,
|
|
42
|
+
readRoles = readModelRoles,
|
|
36
43
|
now,
|
|
37
44
|
} = {}) {
|
|
38
|
-
let current = { backends: {}, choices: {} };
|
|
45
|
+
let current = { backends: {}, choices: {}, found: {} };
|
|
46
|
+
|
|
47
|
+
// A backend whose key did not change keeps its cache. A working Codex stays; one that was not ready
|
|
48
|
+
// is checked again, since the listener may have signed in since.
|
|
49
|
+
async function backendFor(name, found, previous) {
|
|
50
|
+
if (name === "codex") {
|
|
51
|
+
const kept = previous?.backends.codex;
|
|
52
|
+
return kept?.publicStatus().state === "configured" ? kept : createCodex({ environment, now });
|
|
53
|
+
}
|
|
54
|
+
const kept = previous?.backends[name] && previous.found[name]?.key === found[name]?.key;
|
|
55
|
+
return kept ? previous.backends[name] : { deepseek: createDeepSeekWebResearch, brave: createBraveWebResearch }[name]({ apiKey: found[name]?.key, fetch, now });
|
|
56
|
+
}
|
|
39
57
|
|
|
40
58
|
async function load(previous) {
|
|
41
59
|
const found = {
|
|
42
60
|
deepseek: await readKey(credentials, environment, keys.deepseek),
|
|
43
61
|
brave: await readKey(credentials, environment, keys.brave),
|
|
44
62
|
};
|
|
63
|
+
const role = (await readRoles(environment).catch(() => ({}))).web_search?.service ?? null;
|
|
45
64
|
const choices = {};
|
|
46
65
|
for (const kind of ["search", "read"]) {
|
|
47
66
|
const wanted = environment[`MOONDOG_WEB_${kind.toUpperCase()}_PROVIDER`]?.trim().toLowerCase();
|
|
48
|
-
if (wanted) choices[kind] = order[kind].includes(wanted) ? { name: wanted,
|
|
67
|
+
if (wanted) choices[kind] = order[kind].includes(wanted) ? { name: wanted, selectedBy: `MOONDOG_WEB_${kind.toUpperCase()}_PROVIDER` } : { name: null, reason: "web_provider_unknown", wanted };
|
|
68
|
+
// Brave only searches, so with Brave chosen a read still goes to the first backend that can.
|
|
69
|
+
else if (role && order[kind].includes(role)) choices[kind] = { name: role, selectedBy: "settings" };
|
|
49
70
|
else choices[kind] = { name: order[kind].find((name) => name !== "codex" && found[name]) ?? "codex" };
|
|
50
71
|
}
|
|
51
72
|
const backends = {};
|
|
52
73
|
for (const name of new Set(Object.values(choices).map((choice) => choice.name).filter(Boolean))) {
|
|
53
|
-
|
|
54
|
-
const create = { deepseek: createDeepSeekWebResearch, brave: createBraveWebResearch }[name];
|
|
55
|
-
if (create) {
|
|
56
|
-
const kept = previous?.backends[name] && previous.found[name]?.key === found[name]?.key;
|
|
57
|
-
backends[name] = kept ? previous.backends[name] : create({ apiKey: found[name]?.key, fetch, now });
|
|
58
|
-
}
|
|
59
|
-
// A working Codex stays; one that was not ready is checked again, since the listener may have signed in since.
|
|
60
|
-
if (name === "codex") {
|
|
61
|
-
const kept = previous?.backends.codex;
|
|
62
|
-
backends.codex = kept?.publicStatus().state === "configured" ? kept : await createCodex({ environment, now });
|
|
63
|
-
}
|
|
74
|
+
backends[name] = await backendFor(name, found, previous);
|
|
64
75
|
}
|
|
65
76
|
return { backends, choices, found };
|
|
66
77
|
}
|
|
67
78
|
|
|
79
|
+
// The other backends a failed call can go to: ones with a key, and Codex once it is signed in.
|
|
80
|
+
async function alternatives(kind, tried) {
|
|
81
|
+
const names = [];
|
|
82
|
+
for (const name of order[kind]) {
|
|
83
|
+
if (name === tried) continue;
|
|
84
|
+
if (name !== "codex" && !current.found[name]) continue;
|
|
85
|
+
current.backends[name] ??= await backendFor(name, current.found, null);
|
|
86
|
+
if (name === "codex" && current.backends.codex.publicStatus().state !== "configured") continue;
|
|
87
|
+
names.push(name);
|
|
88
|
+
}
|
|
89
|
+
return names;
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
async function research(kind, input, options) {
|
|
93
|
+
const first = backend(kind);
|
|
94
|
+
const name = current.choices[kind].name;
|
|
95
|
+
let failed;
|
|
96
|
+
try {
|
|
97
|
+
const result = await first[kind](input, options);
|
|
98
|
+
// A page the chosen backend could not read may open on another.
|
|
99
|
+
if (kind === "search" || result.status !== "unavailable") return result;
|
|
100
|
+
failed = result;
|
|
101
|
+
} catch (error) {
|
|
102
|
+
if (final.has(error?.code) || options?.signal?.aborted) throw error;
|
|
103
|
+
failed = error;
|
|
104
|
+
}
|
|
105
|
+
for (const other of await alternatives(kind, name)) {
|
|
106
|
+
try {
|
|
107
|
+
const result = await current.backends[other][kind](input, options);
|
|
108
|
+
if (kind === "search" || result.status !== "unavailable") return { ...result, fallback_from: providerIds[name] };
|
|
109
|
+
} catch (error) {
|
|
110
|
+
if (final.has(error?.code) || options?.signal?.aborted) throw error;
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
if (failed instanceof Error) throw failed;
|
|
114
|
+
return failed;
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
/** Each service, whether it is set up for each operation, and which one the role chose; for Settings. */
|
|
118
|
+
async function services() {
|
|
119
|
+
const role = (await readRoles(environment).catch(() => ({}))).web_search?.service ?? null;
|
|
120
|
+
current.backends.codex ??= await backendFor("codex", current.found, null);
|
|
121
|
+
const codex = current.backends.codex.publicStatus().state === "configured";
|
|
122
|
+
return {
|
|
123
|
+
chosen: role,
|
|
124
|
+
search: operationStatus("search"),
|
|
125
|
+
read: operationStatus("read"),
|
|
126
|
+
services: order.search.map((name) => {
|
|
127
|
+
const ready = name === "codex" ? codex : Boolean(current.found[name]);
|
|
128
|
+
return { id: name, provider: providerIds[name], name: webProviderName(providerIds[name]), ready,
|
|
129
|
+
search: ready, read: ready && order.read.includes(name) };
|
|
130
|
+
}),
|
|
131
|
+
};
|
|
132
|
+
}
|
|
133
|
+
|
|
68
134
|
function operationStatus(kind) {
|
|
69
135
|
const choice = current.choices[kind];
|
|
70
136
|
if (!choice?.name) return { provider: null, state: "unavailable", reason: choice?.reason ?? "web_provider_unknown", ...(choice?.wanted ? { requested: choice.wanted } : {}) };
|
|
71
137
|
const provider = providerIds[choice.name];
|
|
72
|
-
const override = choice.
|
|
138
|
+
const override = choice.selectedBy ? { selected_by: choice.selectedBy } : {};
|
|
73
139
|
if (choice.name === "codex") {
|
|
74
140
|
const codex = current.backends.codex.publicStatus();
|
|
75
141
|
return { provider, state: codex.state, ...(codex.reason ? { reason: codex.reason } : {}), ...override };
|
|
@@ -103,8 +169,9 @@ export async function createWebResearch({
|
|
|
103
169
|
current = await load();
|
|
104
170
|
return {
|
|
105
171
|
publicStatus,
|
|
106
|
-
|
|
107
|
-
|
|
172
|
+
services,
|
|
173
|
+
search: (input, options) => research("search", input, options),
|
|
174
|
+
read: (input, options) => research("read", input, options),
|
|
108
175
|
/** Reads keys and settings again, after /auth or /reload. */
|
|
109
176
|
async refresh() {
|
|
110
177
|
const previous = current;
|
|
@@ -4,8 +4,12 @@ import Ajv from "ajv";
|
|
|
4
4
|
|
|
5
5
|
// Every web backend returns this one result shape, checked by the same rules,
|
|
6
6
|
// so the agent and the screen never need to know which service answered.
|
|
7
|
+
// After Hermes Agent's web tools: a search gives five sources unless the agent asks for up to ten,
|
|
8
|
+
// and a read keeps the page's whole text, handing it out a window at a time; the agent asks for the
|
|
9
|
+
// next window with next_offset instead of losing everything past the first.
|
|
7
10
|
export const webLimits = Object.freeze({
|
|
8
|
-
sources: 5,
|
|
11
|
+
sources: 5, maxSources: 10, snippetChars: 1200, textChars: 15_000, fullTextChars: 200_000,
|
|
12
|
+
cacheMs: 300_000, cacheEntries: 32, responseBytes: 2_000_000,
|
|
9
13
|
});
|
|
10
14
|
|
|
11
15
|
// What each backend's text is, so the agent can tell a search engine's snippet from a model's summary.
|
|
@@ -19,13 +23,14 @@ export function webProviderName(provider) {
|
|
|
19
23
|
return backends[provider]?.name ?? null;
|
|
20
24
|
}
|
|
21
25
|
|
|
22
|
-
|
|
26
|
+
// A backend's result holds a page's whole text; the agent's holds one window of it.
|
|
27
|
+
const resultSchema = (textChars) => ({
|
|
23
28
|
type: "object", additionalProperties: false,
|
|
24
29
|
properties: {
|
|
25
30
|
status: { type: "string", enum: ["found", "not_found", "unavailable"] },
|
|
26
|
-
summary: { type: "string", maxLength:
|
|
31
|
+
summary: { type: "string", maxLength: textChars },
|
|
27
32
|
sources: {
|
|
28
|
-
type: "array", maxItems: webLimits.
|
|
33
|
+
type: "array", maxItems: webLimits.maxSources,
|
|
29
34
|
items: {
|
|
30
35
|
type: "object", additionalProperties: false,
|
|
31
36
|
properties: {
|
|
@@ -39,8 +44,11 @@ const resultSchema = {
|
|
|
39
44
|
},
|
|
40
45
|
},
|
|
41
46
|
required: ["status", "summary", "sources"],
|
|
42
|
-
};
|
|
43
|
-
const
|
|
47
|
+
});
|
|
48
|
+
const ajv = new Ajv({ strict: false });
|
|
49
|
+
const validate = ajv.compile(resultSchema(webLimits.fullTextChars));
|
|
50
|
+
const validateWindow = ajv.compile(resultSchema(webLimits.textChars));
|
|
51
|
+
const offsetOf = (value) => (Number.isInteger(value) && value >= 0 ? value : null);
|
|
44
52
|
|
|
45
53
|
export function failure(code, message) {
|
|
46
54
|
return Object.assign(new Error(message), { code });
|
|
@@ -101,7 +109,7 @@ export function webResult({ provider, kind, status, summary = "", sources = [],
|
|
|
101
109
|
export function projectWebResearchResult(value) {
|
|
102
110
|
const payload = { status: value?.status, summary: value?.summary, sources: value?.sources };
|
|
103
111
|
const backend = backends[value?.provider];
|
|
104
|
-
if (!
|
|
112
|
+
if (!validateWindow(payload) || (payload.status === "found" && !payload.sources.length) ||
|
|
105
113
|
!backend?.content[value.kind] || value.evidence !== backend.evidence || !Number.isFinite(Date.parse(value.retrieved_at))) {
|
|
106
114
|
throw failure("web_output_invalid", `${backend?.name ?? "The web service"} returned an invalid or unsourced research result.`);
|
|
107
115
|
}
|
|
@@ -110,6 +118,9 @@ export function projectWebResearchResult(value) {
|
|
|
110
118
|
retrieved_at: new Date(value.retrieved_at).toISOString(), from_cache: value.from_cache === true,
|
|
111
119
|
content_kind: backend.content[value.kind], trust: "untrusted_web_content", evidence: backend.evidence,
|
|
112
120
|
...(value.truncated === true ? { truncated: true } : {}),
|
|
121
|
+
...(value.kind === "read" && offsetOf(value.total_chars) !== null ? {
|
|
122
|
+
text_offset: offsetOf(value.text_offset) ?? 0, total_chars: value.total_chars, next_offset: offsetOf(value.next_offset),
|
|
123
|
+
} : {}),
|
|
113
124
|
summary: cleanText(payload.summary), sources: projectSources(payload.sources),
|
|
114
125
|
};
|
|
115
126
|
}
|
|
@@ -131,14 +142,46 @@ export function searchSources(items) {
|
|
|
131
142
|
if (snippet) seen.add(snippet);
|
|
132
143
|
const date = typeof item.published_at === "string" ? cleanText(item.published_at).slice(0, 100) : "";
|
|
133
144
|
sources.push({ title, url, snippet, published_at: date || null });
|
|
134
|
-
if (sources.length === webLimits.
|
|
145
|
+
if (sources.length === webLimits.maxSources) break;
|
|
135
146
|
}
|
|
136
147
|
return sources;
|
|
137
148
|
}
|
|
138
149
|
|
|
150
|
+
// One window of a page's text, ending at a line or word break near the limit when there is one.
|
|
151
|
+
function textWindow(text, offset) {
|
|
152
|
+
let end = Math.min(text.length, offset + webLimits.textChars);
|
|
153
|
+
if (end < text.length) {
|
|
154
|
+
const floor = offset + Math.floor(webLimits.textChars * 0.8);
|
|
155
|
+
const lineBreak = text.lastIndexOf("\n", end);
|
|
156
|
+
const space = text.lastIndexOf(" ", end);
|
|
157
|
+
end = lineBreak > floor ? lineBreak : space > floor ? space : end;
|
|
158
|
+
}
|
|
159
|
+
return { summary: text.slice(offset, end), end };
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
/** What one call asked for: up to `limit` sources of a search, or the window of a read from `offset`. */
|
|
163
|
+
function windowed(result, kind, { limit, offset }) {
|
|
164
|
+
if (kind === "search") return { ...result, sources: result.sources.slice(0, limit) };
|
|
165
|
+
const text = result.summary;
|
|
166
|
+
if (offset > 0 && offset >= text.length) {
|
|
167
|
+
throw failure("web_input_invalid", `The page has ${text.length} characters, so offset ${offset} is past its end.`);
|
|
168
|
+
}
|
|
169
|
+
const { summary, end } = textWindow(text, offset);
|
|
170
|
+
const more = end < text.length;
|
|
171
|
+
return { ...result, summary, text_offset: offset, total_chars: text.length, next_offset: more ? end : null,
|
|
172
|
+
...(more || result.truncated ? { truncated: true } : {}) };
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
function integerIn(value, minimum, maximum, fallback, message) {
|
|
176
|
+
if (value === undefined) return fallback;
|
|
177
|
+
if (!Number.isInteger(value) || value < minimum || value > maximum) throw failure("web_input_invalid", message);
|
|
178
|
+
return value;
|
|
179
|
+
}
|
|
180
|
+
|
|
139
181
|
/**
|
|
140
182
|
* The parts every backend shares: one query or URL in, a cancellable request,
|
|
141
|
-
* and a five-minute process-local cache of found results.
|
|
183
|
+
* and a five-minute process-local cache of found results. The cache keeps the whole result, so a
|
|
184
|
+
* smaller limit or a later window of the same page needs no second request.
|
|
142
185
|
*/
|
|
143
186
|
export function createWebOperations({ unavailable = () => null, queryChars = 500, run, now = () => new Date() }) {
|
|
144
187
|
const cache = new Map();
|
|
@@ -146,9 +189,14 @@ export function createWebOperations({ unavailable = () => null, queryChars = 500
|
|
|
146
189
|
async function research(kind, input, { signal } = {}) {
|
|
147
190
|
const blocked = unavailable(kind);
|
|
148
191
|
if (blocked) throw blocked;
|
|
149
|
-
|
|
192
|
+
const allowed = kind === "search" ? ["query", "limit"] : ["url", "offset"];
|
|
193
|
+
if (!input || typeof input !== "object" || Object.keys(input).some((key) => !allowed.includes(key))) {
|
|
150
194
|
throw failure("web_input_invalid", "Supply one search query or public URL.");
|
|
151
195
|
}
|
|
196
|
+
const view = {
|
|
197
|
+
limit: integerIn(input.limit, 1, webLimits.maxSources, webLimits.sources, `Ask for 1 to ${webLimits.maxSources} search results.`),
|
|
198
|
+
offset: integerIn(input.offset, 0, webLimits.fullTextChars, 0, "The offset must be a whole number of characters, from 0."),
|
|
199
|
+
};
|
|
152
200
|
const value = kind === "read" ? publicWebUrl(input.url) : typeof input.query === "string" ? input.query.trim() : input.query;
|
|
153
201
|
if (typeof value !== "string" || !value) throw failure("web_input_invalid", "The web query or URL is empty.");
|
|
154
202
|
if (value.length > (kind === "search" ? queryChars : 2048)) {
|
|
@@ -158,7 +206,7 @@ export function createWebOperations({ unavailable = () => null, queryChars = 500
|
|
|
158
206
|
const key = `${kind}:${value}`;
|
|
159
207
|
const cached = cache.get(key);
|
|
160
208
|
if (cached && now().getTime() - Date.parse(cached.retrieved_at) < webLimits.cacheMs) {
|
|
161
|
-
return { ...structuredClone(cached), from_cache: true };
|
|
209
|
+
return windowed({ ...structuredClone(cached), from_cache: true }, kind, view);
|
|
162
210
|
}
|
|
163
211
|
const controller = new AbortController();
|
|
164
212
|
active.add(controller);
|
|
@@ -168,7 +216,7 @@ export function createWebOperations({ unavailable = () => null, queryChars = 500
|
|
|
168
216
|
cache.set(key, result);
|
|
169
217
|
if (cache.size > webLimits.cacheEntries) cache.delete(cache.keys().next().value);
|
|
170
218
|
}
|
|
171
|
-
return structuredClone(result);
|
|
219
|
+
return windowed(structuredClone(result), kind, view);
|
|
172
220
|
} finally {
|
|
173
221
|
active.delete(controller);
|
|
174
222
|
}
|
|
@@ -844,6 +844,7 @@ export class ListeningHistoryStore {
|
|
|
844
844
|
#coverageSources = null;
|
|
845
845
|
#evidence = null;
|
|
846
846
|
#ledgerChanges = 0;
|
|
847
|
+
#listeningDays = null;
|
|
847
848
|
|
|
848
849
|
constructor(database) {
|
|
849
850
|
this.#database = database;
|
|
@@ -2058,6 +2059,59 @@ export class ListeningHistoryStore {
|
|
|
2058
2059
|
...(started.length ? { moondog_started_plays: started.length } : {}), top_artists: artists });
|
|
2059
2060
|
}
|
|
2060
2061
|
|
|
2062
|
+
// The listener's own plays a local day at a time, for the window's taste record: every day's plays, the
|
|
2063
|
+
// same for the most played artists, and the plays Moondog started, kept apart as recentAttention keeps
|
|
2064
|
+
// them. Counts and artist names only. Kept until the stored plays change, since a long history is many rows.
|
|
2065
|
+
listeningDays({ subjectId, artists = 8 } = {}) {
|
|
2066
|
+
if (this.#closed) throw new Error("Listening history store is closed");
|
|
2067
|
+
if (!isUuid(subjectId)) throw new TypeError("A valid profile subject is required");
|
|
2068
|
+
if (!Number.isSafeInteger(artists) || artists < 0 || artists > 20) throw new TypeError("Artist count is invalid");
|
|
2069
|
+
const subject = subjectId.toLowerCase();
|
|
2070
|
+
const version = `${this.listeningDataVersion()}:${subject}:${artists}`;
|
|
2071
|
+
if (this.#listeningDays?.version === version) return structuredClone(this.#listeningDays.value);
|
|
2072
|
+
const effective = `e.subject_id = ?
|
|
2073
|
+
AND NOT EXISTS (SELECT 1 FROM listening_event_supersessions s WHERE s.superseded_event_id = e.listening_event_id)
|
|
2074
|
+
AND NOT EXISTS (SELECT 1 FROM listening_event_mirrors m WHERE m.mirror_event_id = e.listening_event_id)`;
|
|
2075
|
+
const suggestions = this.#suggestionsForMatching();
|
|
2076
|
+
const since = suggestions.length ? new Date(Date.parse(suggestions[0].requested_at) - SUGGESTION_WINDOW.beforeMs).toISOString() : null;
|
|
2077
|
+
const started = !since ? [] : markMoondogStartedPlays(this.#database.prepare(`SELECT
|
|
2078
|
+
e.listening_event_id, e.record_json AS event_json, t.record_json AS track_json FROM listening_events e
|
|
2079
|
+
JOIN track_refs t ON t.track_ref_id = e.track_ref_id AND t.revision = e.track_ref_revision
|
|
2080
|
+
WHERE ${effective} AND e.occurred_at >= ?`)
|
|
2081
|
+
.all(subject, since).map((row) => ({ id: row.listening_event_id, event: JSON.parse(row.event_json), track: JSON.parse(row.track_json) })),
|
|
2082
|
+
suggestions).rows.filter((row) => row.moondog_suggestion_id).map((row) => row.id);
|
|
2083
|
+
const own = `${effective} AND e.listening_event_id NOT IN (SELECT value FROM json_each(?))`;
|
|
2084
|
+
const left = JSON.stringify(started);
|
|
2085
|
+
const pairs = (rows) => rows.map((row) => [row.day, row.plays]);
|
|
2086
|
+
const days = this.#database.prepare(`SELECT date(e.occurred_at, 'localtime') AS day, COUNT(*) AS plays
|
|
2087
|
+
FROM listening_events e WHERE ${own} GROUP BY day ORDER BY day`).all(subject, left);
|
|
2088
|
+
let value = null;
|
|
2089
|
+
if (days.length) {
|
|
2090
|
+
const credited = `FROM listening_events e
|
|
2091
|
+
JOIN track_refs t ON t.track_ref_id = e.track_ref_id AND t.revision = e.track_ref_revision
|
|
2092
|
+
JOIN json_each(t.record_json, '$.artist_credits') a
|
|
2093
|
+
WHERE ${own} AND json_extract(a.value, '$.name') IS NOT NULL`;
|
|
2094
|
+
const top = artists === 0 ? [] : this.#database.prepare(`SELECT json_extract(a.value, '$.name') AS name, COUNT(*) AS plays
|
|
2095
|
+
${credited} GROUP BY name ORDER BY plays DESC, name LIMIT ?`).all(subject, left, artists);
|
|
2096
|
+
const byArtist = new Map(top.map((row) => [row.name, []]));
|
|
2097
|
+
if (top.length) {
|
|
2098
|
+
for (const row of this.#database.prepare(`SELECT json_extract(a.value, '$.name') AS name,
|
|
2099
|
+
date(e.occurred_at, 'localtime') AS day, COUNT(*) AS plays
|
|
2100
|
+
${credited} AND json_extract(a.value, '$.name') IN (SELECT value FROM json_each(?))
|
|
2101
|
+
GROUP BY name, day ORDER BY day`).iterate(subject, left, JSON.stringify(top.map((artist) => artist.name)))) {
|
|
2102
|
+
byArtist.get(row.name)?.push([row.day, row.plays]);
|
|
2103
|
+
}
|
|
2104
|
+
}
|
|
2105
|
+
const moondog = started.length ? this.#database.prepare(`SELECT date(occurred_at, 'localtime') AS day, COUNT(*) AS plays
|
|
2106
|
+
FROM listening_events WHERE listening_event_id IN (SELECT value FROM json_each(?)) GROUP BY day ORDER BY day`).all(left) : [];
|
|
2107
|
+
value = { first_day: days[0].day, last_day: days.at(-1).day, days: pairs(days),
|
|
2108
|
+
artists: top.map((artist) => ({ name: artist.name, plays: artist.plays, days: byArtist.get(artist.name) })),
|
|
2109
|
+
moondog_days: pairs(moondog) };
|
|
2110
|
+
}
|
|
2111
|
+
this.#listeningDays = { version, value };
|
|
2112
|
+
return structuredClone(value);
|
|
2113
|
+
}
|
|
2114
|
+
|
|
2061
2115
|
// Which spans of time the stored plays cover, and where plays may be missing.
|
|
2062
2116
|
listeningCoverage({ subjectId } = {}) {
|
|
2063
2117
|
if (this.#closed) throw new Error("Listening history store is closed");
|