@audiofool/moondog 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +125 -0
- package/assets/brand/moondog-lunar-record/moondog-logo-1x1.png +0 -0
- package/contracts/v1/capability-policy.schema.json +572 -0
- package/contracts/v1/definitions.schema.json +365 -0
- package/contracts/v1/library-track-observation.schema.json +138 -0
- package/contracts/v1/listening-event.schema.json +111 -0
- package/contracts/v1/profile-evidence.schema.json +237 -0
- package/contracts/v1/taste-event.schema.json +258 -0
- package/contracts/v1/tool-invocation.schema.json +318 -0
- package/contracts/v1/tool-receipt.schema.json +1066 -0
- package/contracts/v1/track-ref.schema.json +275 -0
- package/contracts/v2/profile-evidence.schema.json +284 -0
- package/package.json +105 -0
- package/scripts/contract-lib.mjs +361 -0
- package/scripts/contract-semantics.mjs +2630 -0
- package/scripts/demo-studio.mjs +58 -0
- package/scripts/moondog +52 -0
- package/scripts/moondog.mjs +954 -0
- package/src/core/apple-data-migration.mjs +50 -0
- package/src/core/apple-library-source-status.mjs +220 -0
- package/src/core/apple-projection-domain-services.mjs +2356 -0
- package/src/core/capability-catalog.mjs +577 -0
- package/src/core/listening-profile-domain-services.mjs +1010 -0
- package/src/core/lyric-library.mjs +121 -0
- package/src/core/lyric-profile.mjs +50 -0
- package/src/core/lyric-service.mjs +41 -0
- package/src/core/moondog-application.mjs +1827 -0
- package/src/core/state-directory.mjs +19 -0
- package/src/core/synthetic-domain-services.mjs +1485 -0
- package/src/core/uuid-v5.mjs +42 -0
- package/src/demo/fictional-spotify-history.mjs +400 -0
- package/src/demo/moondog-demo.mjs +355 -0
- package/src/demo/moondog-interactive-tasteprint-demo.mjs +293 -0
- package/src/demo/moondog-tasteprint-demo.mjs +631 -0
- package/src/importers/apple-music-library/import-batch.mjs +290 -0
- package/src/importers/apple-music-library/index.mjs +76 -0
- package/src/importers/apple-music-library/normalize.mjs +1126 -0
- package/src/importers/apple-music-library/parse-plist.mjs +357 -0
- package/src/importers/apple-music-library/projection-records.mjs +557 -0
- package/src/importers/apple-music-library/read-batch.mjs +276 -0
- package/src/importers/apple-music-library/sqlite-projection.mjs +1307 -0
- package/src/importers/apple-music-library/stable-ids.mjs +19 -0
- package/src/importers/apple-music-library/track-snapshot.schema.json +136 -0
- package/src/integrations/apple-music/catalog.mjs +770 -0
- package/src/integrations/cross-catalog-artist-identity.mjs +222 -0
- package/src/integrations/listenbrainz/history-file.mjs +560 -0
- package/src/integrations/lyrics/lrclib.mjs +55 -0
- package/src/integrations/open-music-similarity/artist-radio.mjs +806 -0
- package/src/integrations/public-playlists.mjs +104 -0
- package/src/integrations/spotify/account-data-history.mjs +545 -0
- package/src/integrations/spotify/account-data-profile.mjs +918 -0
- package/src/integrations/spotify/authentication.mjs +791 -0
- package/src/integrations/spotify/catalog-resolver.mjs +657 -0
- package/src/integrations/spotify/connection.mjs +125 -0
- package/src/integrations/spotify/credential-store.mjs +26 -0
- package/src/integrations/spotify/extended-streaming-history.mjs +525 -0
- package/src/integrations/spotify/history-archive.mjs +59 -0
- package/src/integrations/spotify/index.mjs +80 -0
- package/src/integrations/spotify/recent-activity.mjs +198 -0
- package/src/integrations/spotify/resolution-cache.mjs +260 -0
- package/src/integrations/spotify/service.mjs +713 -0
- package/src/integrations/spotify/settings.mjs +89 -0
- package/src/integrations/spotify/web-api-client.mjs +761 -0
- package/src/integrations/web/codex-web.mjs +263 -0
- package/src/integrations/youtube-music/takeout.mjs +143 -0
- package/src/memory/local-memory-store.mjs +1705 -0
- package/src/memory/reflection-worker.mjs +87 -0
- package/src/profile/apple-library-import.mjs +28 -0
- package/src/profile/history-import.mjs +163 -0
- package/src/profile/listener-corrections.mjs +217 -0
- package/src/profile/listening-history-store.mjs +2055 -0
- package/src/profile/listening-profile-projection.mjs +2520 -0
- package/src/profile/local-music-data.mjs +812 -0
- package/src/profile/music-import-bundle.mjs +91 -0
- package/src/profile/music-providers.mjs +25 -0
- package/src/profile/spotify-archive-catalog-hints.mjs +210 -0
- package/src/profile/spotify-archive-taste.mjs +482 -0
- package/src/runtime/pi/agent-runtime.mjs +6444 -0
- package/src/runtime/pi/authentication.mjs +104 -0
- package/src/runtime/pi/configured-runtime.mjs +98 -0
- package/src/runtime/pi/memory-agent-runtime.mjs +337 -0
- package/src/runtime/pi/model-catalog.mjs +62 -0
- package/src/runtime/pi/model-transport.mjs +117 -0
- package/src/runtime/pi/persistent-credential-store.mjs +670 -0
- package/src/runtime/pi/provider-registry.mjs +74 -0
- package/src/runtime/pi/runtime-settings.mjs +81 -0
- package/src/surfaces/cli/auth-command.mjs +355 -0
- package/src/surfaces/cli/brand-components.mjs +390 -0
- package/src/surfaces/cli/brand-theme.mjs +231 -0
- package/src/surfaces/cli/catalog-command.mjs +509 -0
- package/src/surfaces/cli/command-completions.mjs +113 -0
- package/src/surfaces/cli/data-command.mjs +214 -0
- package/src/surfaces/cli/format-output.mjs +750 -0
- package/src/surfaces/cli/history-import-view.mjs +445 -0
- package/src/surfaces/cli/import-guides.mjs +9 -0
- package/src/surfaces/cli/listenbrainz-command.mjs +129 -0
- package/src/surfaces/cli/listening-menu.mjs +171 -0
- package/src/surfaces/cli/profile-command.mjs +290 -0
- package/src/surfaces/cli/spotify-command.mjs +849 -0
- package/src/surfaces/cli/taste-profile-model.mjs +306 -0
- package/src/surfaces/cli/taste-profile-view.mjs +272 -0
- package/src/surfaces/cli/terminal-art.mjs +295 -0
- package/src/surfaces/cli/tui.mjs +1714 -0
- package/src/surfaces/cli/web-command.mjs +52 -0
- package/src/surfaces/html/tasteprint-artifact.mjs +96 -0
- package/src/surfaces/html/tasteprint.mjs +3575 -0
- package/src/surfaces/web/showcase.mjs +272 -0
- package/src/surfaces/web/studio-page.mjs +4148 -0
- package/src/surfaces/web/studio.mjs +2288 -0
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
import { musicHash, musicText, prepareMusicBundle } from "../profile/music-import-bundle.mjs";
|
|
2
|
+
|
|
3
|
+
const qqHosts = ["y.qq.com", "c.y.qq.com", "i.y.qq.com", "m.y.qq.com"];
|
|
4
|
+
const hosts = new Set([...qqHosts, "music.163.com", "y.music.163.com", "163cn.tv"]);
|
|
5
|
+
const numericId = /^[1-9][0-9]{0,19}$/u;
|
|
6
|
+
|
|
7
|
+
function allowedUrl(value) {
|
|
8
|
+
let url;
|
|
9
|
+
try { url = new URL(value); } catch { throw new Error("Paste a QQ Music or NetEase Cloud Music playlist share link."); }
|
|
10
|
+
if (!hosts.has(url.hostname) || !["http:", "https:"].includes(url.protocol) || url.username || url.password || url.port) {
|
|
11
|
+
throw new Error("Use a playlist link from y.qq.com, music.163.com or their official share links.");
|
|
12
|
+
}
|
|
13
|
+
url.protocol = "https:";
|
|
14
|
+
return url;
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
export function playlistLink(input) {
|
|
18
|
+
const links = String(input).match(/https?:\/\/[^\s<>"“”]+/gu) ?? [];
|
|
19
|
+
if (links.length !== 1) throw new Error("Paste one playlist share link at a time.");
|
|
20
|
+
return allowedUrl(links[0]);
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
function playlistIdentity(url) {
|
|
24
|
+
const fragment = url.hash.startsWith("#/") ? new URL(url.hash.slice(1), url.origin) : url;
|
|
25
|
+
if (["music.163.com", "y.music.163.com"].includes(url.hostname)) {
|
|
26
|
+
const id = fragment.searchParams.get("id");
|
|
27
|
+
if (/(?:^|\/)playlist\/?$/u.test(fragment.pathname) && numericId.test(id ?? "")) return { provider: "netease", id };
|
|
28
|
+
}
|
|
29
|
+
if (qqHosts.includes(url.hostname)) {
|
|
30
|
+
const match = url.pathname.match(/^\/n\/(?:ryqq|ryqq_v2)\/playlist\/([0-9]+)\/?$/u);
|
|
31
|
+
const id = match?.[1] ?? url.searchParams.get("disstid") ?? url.searchParams.get("id");
|
|
32
|
+
if ((match || /(?:taoge|playlist|playsquare)/u.test(url.pathname)) && numericId.test(id ?? "")) return { provider: "qq_music", id };
|
|
33
|
+
}
|
|
34
|
+
return null;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
async function responseFor(url, fetchImpl, referer) {
|
|
38
|
+
try {
|
|
39
|
+
return await fetchImpl(url, { redirect: "manual", signal: AbortSignal.timeout(15_000),
|
|
40
|
+
headers: { "User-Agent": "Mozilla/5.0 (compatible; Moondog music metadata import)", ...(referer ? { Referer: referer } : {}) } });
|
|
41
|
+
} catch { throw new Error("Could not reach the music service. Check your connection and try the same playlist link again."); }
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
async function boundedJson(response) {
|
|
45
|
+
if (!response.ok) { await response.body?.cancel(); throw new Error("The service did not return a public playlist. Open the link in your browser to check that it is available without signing in."); }
|
|
46
|
+
const chunks = [];
|
|
47
|
+
let size = 0;
|
|
48
|
+
for await (const chunk of response.body) {
|
|
49
|
+
size += chunk.length;
|
|
50
|
+
if (size > 16 * 1024 * 1024) throw new Error("Playlist response is too large. Choose a smaller playlist.");
|
|
51
|
+
chunks.push(chunk);
|
|
52
|
+
}
|
|
53
|
+
try { return JSON.parse(Buffer.concat(chunks).toString("utf8")); }
|
|
54
|
+
catch { throw new Error("The service changed its playlist response or requires sign-in. No music was imported."); }
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
export async function readPublicPlaylist({ input, subjectId, capturedAt, provider, fetchImpl = fetch }) {
|
|
58
|
+
let url = playlistLink(input);
|
|
59
|
+
let identity = playlistIdentity(url);
|
|
60
|
+
// Follow only official share redirects, never arbitrary network destinations or cookies.
|
|
61
|
+
for (let index = 0; !identity && index < 4; index += 1) {
|
|
62
|
+
const response = await responseFor(url, fetchImpl);
|
|
63
|
+
const location = response.headers.get("location");
|
|
64
|
+
await response.body?.cancel();
|
|
65
|
+
if (response.status < 300 || response.status >= 400 || !location) break;
|
|
66
|
+
url = allowedUrl(new URL(location, url).href);
|
|
67
|
+
identity = playlistIdentity(url);
|
|
68
|
+
}
|
|
69
|
+
if (!identity) throw new Error("Open this share link in your browser, then copy the full playlist page URL. Song, album and private playlist links are not supported.");
|
|
70
|
+
if (provider && ["qq_music", "netease"].includes(provider) && provider !== identity.provider) throw new Error("This link belongs to another service. Go back and choose that service, or paste the matching playlist link.");
|
|
71
|
+
const qq = identity.provider === "qq_music";
|
|
72
|
+
const endpoint = qq
|
|
73
|
+
? `https://c.y.qq.com/qzone/fcg-bin/fcg_ucc_getcdinfo_byids_cp.fcg?type=1&json=1&utf8=1&onlysong=0&disstid=${identity.id}&format=json`
|
|
74
|
+
: `https://music.163.com/api/playlist/detail?id=${identity.id}`;
|
|
75
|
+
const data = await boundedJson(await responseFor(endpoint, fetchImpl, qq ? "https://y.qq.com/" : "https://music.163.com/"));
|
|
76
|
+
const playlist = qq ? data?.cdlist?.[0] : data?.result;
|
|
77
|
+
if ((qq ? data?.code !== 0 || data?.subcode !== 0 : data?.code !== 200) || !playlist
|
|
78
|
+
|| (qq ? Number(playlist.dir_show) === 0 : Number(playlist.privacy) === 10)) throw new Error("This playlist is unavailable publicly. Choose a playlist you can open without signing in; no account password or cookies are needed.");
|
|
79
|
+
if (String(qq ? playlist.disstid : playlist.id) !== identity.id) throw new Error("The service returned a different playlist. Nothing imported; try copying the playlist link again.");
|
|
80
|
+
const songs = qq ? playlist.songlist : playlist.tracks;
|
|
81
|
+
if (!Array.isArray(songs) || songs.length > 10_000) throw new Error("The playlist format is unsupported or exceeds 10,000 songs.");
|
|
82
|
+
const name = musicText(qq ? playlist.dissname : playlist.name) || "Selected playlist";
|
|
83
|
+
const totalValue = Number(qq ? playlist.total_song_num ?? playlist.songnum : playlist.trackCount);
|
|
84
|
+
const total = Number.isSafeInteger(totalValue) && totalValue >= songs.length ? totalValue : null;
|
|
85
|
+
const rows = songs.map((song, index) => {
|
|
86
|
+
if (!song || typeof song !== "object") return {};
|
|
87
|
+
const artists = qq ? song.singer : song.artists ?? song.ar;
|
|
88
|
+
return {
|
|
89
|
+
id: qq ? song.songmid || String(song.songid ?? "") : String(song.id ?? ""),
|
|
90
|
+
title: musicText(qq ? song.songname : song.name),
|
|
91
|
+
artist: musicText(Array.isArray(artists) ? artists.map((artist) => musicText(artist?.name)).filter(Boolean).join(", ") : ""),
|
|
92
|
+
album: musicText(qq ? song.albumname : (song.album ?? song.al)?.name),
|
|
93
|
+
playlistId: identity.id, playlistName: name, position: index + 1, member: "public-playlist.json",
|
|
94
|
+
};
|
|
95
|
+
}).filter((row) => row.id && row.title);
|
|
96
|
+
const bytes = Buffer.from(JSON.stringify({ provider: identity.provider, id: identity.id, name, rows }));
|
|
97
|
+
const prepared = prepareMusicBundle({ provider: identity.provider, subjectId, capturedAt, rows,
|
|
98
|
+
digest: musicHash(bytes), size: bytes.length, members: ["public-playlist.json"], fileName: name, skipped: songs.length - rows.length,
|
|
99
|
+
scopeNote: `${rows.length} of ${total ?? "an unknown number of"} songs could be read; the service hides some. Only this playlist is added. It shows what you keep, not what you played, and it may not be one you made.`,
|
|
100
|
+
});
|
|
101
|
+
prepared.preview.availableTracks = rows.length;
|
|
102
|
+
prepared.preview.totalTracks = total;
|
|
103
|
+
return prepared;
|
|
104
|
+
}
|
|
@@ -0,0 +1,545 @@
|
|
|
1
|
+
import { execFile } from "node:child_process";
|
|
2
|
+
import { createHash } from "node:crypto";
|
|
3
|
+
import { createReadStream } from "node:fs";
|
|
4
|
+
import { lstat } from "node:fs/promises";
|
|
5
|
+
import path from "node:path";
|
|
6
|
+
import { promisify } from "node:util";
|
|
7
|
+
|
|
8
|
+
import { isUuid, uuidV5 } from "../../core/uuid-v5.mjs";
|
|
9
|
+
import {
|
|
10
|
+
SPOTIFY_ACCOUNT_PROFILE_MEMBERS,
|
|
11
|
+
mergeSpotifyAccountProfile,
|
|
12
|
+
projectSpotifyAccountDataProfile,
|
|
13
|
+
} from "./account-data-profile.mjs";
|
|
14
|
+
|
|
15
|
+
export const SPOTIFY_ACCOUNT_DATA_HISTORY_SOURCE =
|
|
16
|
+
"spotify.account_data.streaming_history";
|
|
17
|
+
export const SPOTIFY_ACCOUNT_DATA_HISTORY_FORMAT =
|
|
18
|
+
"spotify_account_data_streaming_history_v1";
|
|
19
|
+
export const SPOTIFY_ACCOUNT_DATA_HISTORY_SCOPE = "past_year_account_data";
|
|
20
|
+
export const SPOTIFY_ACCOUNT_DATA_TRACK_NAMESPACE =
|
|
21
|
+
"a3ac8a61-684b-58e0-8bb0-c9691b9323e7";
|
|
22
|
+
export const SPOTIFY_ACCOUNT_DATA_EVENT_NAMESPACE =
|
|
23
|
+
"18f7b1ad-0da4-51f9-a023-2807e7b8d5cf";
|
|
24
|
+
export const SPOTIFY_ACCOUNT_DATA_BATCH_NAMESPACE =
|
|
25
|
+
"47382c18-99ba-5d22-a1e1-4adf89145cdb";
|
|
26
|
+
|
|
27
|
+
export const SPOTIFY_ACCOUNT_DATA_HISTORY_LIMITS = Object.freeze({
|
|
28
|
+
maximumArchiveBytes: 512 * 1024 * 1024,
|
|
29
|
+
maximumHistoryMembers: 100,
|
|
30
|
+
maximumMemberBytes: 64 * 1024 * 1024,
|
|
31
|
+
maximumTotalJsonBytes: 128 * 1024 * 1024,
|
|
32
|
+
maximumRecords: 500_000,
|
|
33
|
+
});
|
|
34
|
+
|
|
35
|
+
const execFileAsync = promisify(execFile);
|
|
36
|
+
const musicMemberPattern =
|
|
37
|
+
/^Spotify Account Data\/StreamingHistory_music_([0-9]+)\.json$/u;
|
|
38
|
+
const endTimePattern =
|
|
39
|
+
/^([0-9]{4}-[0-9]{2}-[0-9]{2}) ([0-9]{2}:[0-9]{2})$/u;
|
|
40
|
+
const unsafeTextPattern =
|
|
41
|
+
/[\u0000-\u001f\u007f-\u009f\u202a-\u202e\u2066-\u2069]/gu;
|
|
42
|
+
const sha256Pattern = /^[a-f0-9]{64}$/u;
|
|
43
|
+
|
|
44
|
+
export class SpotifyAccountDataHistoryError extends Error {
|
|
45
|
+
constructor(message, code = "spotify_account_data_invalid") {
|
|
46
|
+
super(message);
|
|
47
|
+
this.name = "SpotifyAccountDataHistoryError";
|
|
48
|
+
this.code = code;
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
function fail(message, code) {
|
|
53
|
+
throw new SpotifyAccountDataHistoryError(message, code);
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
function normalizedTimestamp(value, label) {
|
|
57
|
+
if (typeof value !== "string") fail(`${label} is invalid.`);
|
|
58
|
+
const milliseconds = Date.parse(value);
|
|
59
|
+
if (!Number.isFinite(milliseconds)) fail(`${label} is invalid.`);
|
|
60
|
+
const normalized = new Date(milliseconds).toISOString();
|
|
61
|
+
if (normalized !== value) fail(`${label} must be a normalized UTC timestamp.`);
|
|
62
|
+
return normalized;
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
function accountDataEndTime(value, index) {
|
|
66
|
+
if (typeof value !== "string") {
|
|
67
|
+
fail(`Spotify music history record ${index + 1} has an invalid endTime.`);
|
|
68
|
+
}
|
|
69
|
+
const match = endTimePattern.exec(value);
|
|
70
|
+
if (!match) {
|
|
71
|
+
fail(`Spotify music history record ${index + 1} has an invalid endTime.`);
|
|
72
|
+
}
|
|
73
|
+
const normalized = `${match[1]}T${match[2]}:00.000Z`;
|
|
74
|
+
const milliseconds = Date.parse(normalized);
|
|
75
|
+
if (
|
|
76
|
+
!Number.isFinite(milliseconds) ||
|
|
77
|
+
new Date(milliseconds).toISOString() !== normalized
|
|
78
|
+
) {
|
|
79
|
+
fail(`Spotify music history record ${index + 1} has an invalid endTime.`);
|
|
80
|
+
}
|
|
81
|
+
return normalized;
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
function cleanText(value, label, index) {
|
|
85
|
+
if (typeof value !== "string") {
|
|
86
|
+
fail(`Spotify music history record ${index + 1} has an invalid ${label}.`);
|
|
87
|
+
}
|
|
88
|
+
const cleaned = value
|
|
89
|
+
.normalize("NFC")
|
|
90
|
+
.replace(unsafeTextPattern, " ")
|
|
91
|
+
.replace(/\s+/gu, " ")
|
|
92
|
+
.trim();
|
|
93
|
+
if (!cleaned || Array.from(cleaned).length > 512) {
|
|
94
|
+
fail(`Spotify music history record ${index + 1} has an invalid ${label}.`);
|
|
95
|
+
}
|
|
96
|
+
return cleaned;
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
function normalizedIdentityText(value) {
|
|
100
|
+
return value.normalize("NFKC").toLowerCase();
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
function hashParts(...parts) {
|
|
104
|
+
const hash = createHash("sha256");
|
|
105
|
+
for (const part of parts) hash.update(part).update("\0");
|
|
106
|
+
return hash.digest("hex");
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
function clippedText(value, maximum = 512) {
|
|
110
|
+
return Array.from(value).slice(0, maximum).join("");
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
function lexicalCompare(left, right) {
|
|
114
|
+
return left < right ? -1 : left > right ? 1 : 0;
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
function accountDataEventTuple({ occurredAt, artist, title, playedMs }) {
|
|
118
|
+
return [
|
|
119
|
+
occurredAt,
|
|
120
|
+
normalizedIdentityText(artist),
|
|
121
|
+
normalizedIdentityText(title),
|
|
122
|
+
String(playedMs),
|
|
123
|
+
].join("\0");
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
export function spotifyAccountDataTrackIdentity({ artist, title } = {}) {
|
|
127
|
+
if (typeof artist !== "string" || typeof title !== "string") {
|
|
128
|
+
fail("Spotify account-data track identity is invalid.");
|
|
129
|
+
}
|
|
130
|
+
const fingerprint = hashParts(
|
|
131
|
+
"spotify-account-data-track-label-v1",
|
|
132
|
+
normalizedIdentityText(artist),
|
|
133
|
+
normalizedIdentityText(title),
|
|
134
|
+
);
|
|
135
|
+
return {
|
|
136
|
+
fingerprint,
|
|
137
|
+
track_ref_id: uuidV5(
|
|
138
|
+
fingerprint,
|
|
139
|
+
SPOTIFY_ACCOUNT_DATA_TRACK_NAMESPACE,
|
|
140
|
+
),
|
|
141
|
+
};
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
export function spotifyAccountDataEventIdentity({
|
|
145
|
+
subjectId,
|
|
146
|
+
occurredAt,
|
|
147
|
+
artist,
|
|
148
|
+
title,
|
|
149
|
+
playedMs,
|
|
150
|
+
duplicateOrdinal,
|
|
151
|
+
} = {}) {
|
|
152
|
+
if (
|
|
153
|
+
!isUuid(subjectId) ||
|
|
154
|
+
typeof artist !== "string" ||
|
|
155
|
+
typeof title !== "string" ||
|
|
156
|
+
!Number.isSafeInteger(playedMs) ||
|
|
157
|
+
playedMs < 0 ||
|
|
158
|
+
!Number.isSafeInteger(duplicateOrdinal) ||
|
|
159
|
+
duplicateOrdinal < 1
|
|
160
|
+
) {
|
|
161
|
+
fail("Spotify account-data event identity is invalid.");
|
|
162
|
+
}
|
|
163
|
+
const normalizedOccurredAt = normalizedTimestamp(
|
|
164
|
+
occurredAt,
|
|
165
|
+
"Spotify account-data event timestamp",
|
|
166
|
+
);
|
|
167
|
+
const tuple = accountDataEventTuple({
|
|
168
|
+
occurredAt: normalizedOccurredAt,
|
|
169
|
+
artist,
|
|
170
|
+
title,
|
|
171
|
+
playedMs,
|
|
172
|
+
});
|
|
173
|
+
const fingerprint = hashParts(
|
|
174
|
+
"spotify-account-data-play-v1",
|
|
175
|
+
subjectId.toLowerCase(),
|
|
176
|
+
tuple,
|
|
177
|
+
String(duplicateOrdinal),
|
|
178
|
+
);
|
|
179
|
+
return {
|
|
180
|
+
fingerprint,
|
|
181
|
+
listening_event_id: uuidV5(
|
|
182
|
+
fingerprint,
|
|
183
|
+
SPOTIFY_ACCOUNT_DATA_EVENT_NAMESPACE,
|
|
184
|
+
),
|
|
185
|
+
tuple,
|
|
186
|
+
};
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
function normalizedMusicRows(records, capturedAt) {
|
|
190
|
+
if (!Array.isArray(records)) {
|
|
191
|
+
fail("Spotify music history records are invalid.");
|
|
192
|
+
}
|
|
193
|
+
if (records.length > SPOTIFY_ACCOUNT_DATA_HISTORY_LIMITS.maximumRecords) {
|
|
194
|
+
fail("Spotify music history contains too many records.");
|
|
195
|
+
}
|
|
196
|
+
const capturedMilliseconds = Date.parse(capturedAt);
|
|
197
|
+
return records.map((record, index) => {
|
|
198
|
+
if (!record || typeof record !== "object" || Array.isArray(record)) {
|
|
199
|
+
fail(`Spotify music history record ${index + 1} is invalid.`);
|
|
200
|
+
}
|
|
201
|
+
const occurredAt = accountDataEndTime(record.endTime, index);
|
|
202
|
+
if (Date.parse(occurredAt) > capturedMilliseconds) {
|
|
203
|
+
fail(`Spotify music history record ${index + 1} is later than the import time.`);
|
|
204
|
+
}
|
|
205
|
+
const artist = cleanText(record.artistName, "artistName", index);
|
|
206
|
+
const title = cleanText(record.trackName, "trackName", index);
|
|
207
|
+
if (!Number.isSafeInteger(record.msPlayed) || record.msPlayed < 0) {
|
|
208
|
+
fail(`Spotify music history record ${index + 1} has an invalid msPlayed.`);
|
|
209
|
+
}
|
|
210
|
+
return {
|
|
211
|
+
artist,
|
|
212
|
+
title,
|
|
213
|
+
occurredAt,
|
|
214
|
+
playedMs: record.msPlayed,
|
|
215
|
+
tuple: accountDataEventTuple({
|
|
216
|
+
occurredAt,
|
|
217
|
+
artist,
|
|
218
|
+
title,
|
|
219
|
+
playedMs: record.msPlayed,
|
|
220
|
+
}),
|
|
221
|
+
};
|
|
222
|
+
});
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
function normalizedMemberNames(memberNames) {
|
|
226
|
+
if (
|
|
227
|
+
!Array.isArray(memberNames) ||
|
|
228
|
+
memberNames.length < 1 ||
|
|
229
|
+
memberNames.length > SPOTIFY_ACCOUNT_DATA_HISTORY_LIMITS.maximumHistoryMembers
|
|
230
|
+
) {
|
|
231
|
+
fail("Spotify music history member names are invalid.");
|
|
232
|
+
}
|
|
233
|
+
const normalized = memberNames.map((value) => {
|
|
234
|
+
if (
|
|
235
|
+
typeof value !== "string" ||
|
|
236
|
+
!/^StreamingHistory_music_[0-9]+\.json$/u.test(value)
|
|
237
|
+
) {
|
|
238
|
+
fail("Spotify music history member names are invalid.");
|
|
239
|
+
}
|
|
240
|
+
return value;
|
|
241
|
+
});
|
|
242
|
+
if (new Set(normalized).size !== normalized.length) {
|
|
243
|
+
fail("Spotify music history contains duplicate member names.");
|
|
244
|
+
}
|
|
245
|
+
return [...normalized].sort(lexicalCompare);
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
export function projectSpotifyAccountDataHistory({
|
|
249
|
+
subjectId,
|
|
250
|
+
records,
|
|
251
|
+
archiveSha256,
|
|
252
|
+
archiveSizeBytes,
|
|
253
|
+
memberNames,
|
|
254
|
+
capturedAt = new Date().toISOString(),
|
|
255
|
+
} = {}) {
|
|
256
|
+
if (!isUuid(subjectId)) fail("A trusted subject ID is required.");
|
|
257
|
+
const normalizedSubjectId = subjectId.toLowerCase();
|
|
258
|
+
const normalizedCapturedAt = normalizedTimestamp(
|
|
259
|
+
capturedAt,
|
|
260
|
+
"Spotify account-data import timestamp",
|
|
261
|
+
);
|
|
262
|
+
if (typeof archiveSha256 !== "string" || !sha256Pattern.test(archiveSha256)) {
|
|
263
|
+
fail("Spotify account-data archive SHA-256 is invalid.");
|
|
264
|
+
}
|
|
265
|
+
if (
|
|
266
|
+
!Number.isSafeInteger(archiveSizeBytes) ||
|
|
267
|
+
archiveSizeBytes < 1 ||
|
|
268
|
+
archiveSizeBytes > SPOTIFY_ACCOUNT_DATA_HISTORY_LIMITS.maximumArchiveBytes
|
|
269
|
+
) {
|
|
270
|
+
fail("Spotify account-data archive size is invalid.");
|
|
271
|
+
}
|
|
272
|
+
const normalizedMembers = normalizedMemberNames(memberNames);
|
|
273
|
+
const rows = normalizedMusicRows(records, normalizedCapturedAt).sort(
|
|
274
|
+
(left, right) => lexicalCompare(left.tuple, right.tuple),
|
|
275
|
+
);
|
|
276
|
+
const importBatchId = uuidV5(
|
|
277
|
+
`${normalizedSubjectId}\0${archiveSha256}`,
|
|
278
|
+
SPOTIFY_ACCOUNT_DATA_BATCH_NAMESPACE,
|
|
279
|
+
);
|
|
280
|
+
const trackRefs = new Map();
|
|
281
|
+
const listeningEvents = [];
|
|
282
|
+
let previousTuple = null;
|
|
283
|
+
let duplicateOrdinal = 0;
|
|
284
|
+
|
|
285
|
+
for (const row of rows) {
|
|
286
|
+
if (row.tuple === previousTuple) duplicateOrdinal += 1;
|
|
287
|
+
else duplicateOrdinal = 1;
|
|
288
|
+
previousTuple = row.tuple;
|
|
289
|
+
|
|
290
|
+
const trackIdentity = spotifyAccountDataTrackIdentity(row);
|
|
291
|
+
const trackFingerprint = trackIdentity.fingerprint;
|
|
292
|
+
const trackRefId = trackIdentity.track_ref_id;
|
|
293
|
+
if (!trackRefs.has(trackRefId)) {
|
|
294
|
+
trackRefs.set(trackRefId, {
|
|
295
|
+
schema_version: 1,
|
|
296
|
+
track_ref_id: trackRefId,
|
|
297
|
+
revision: 1,
|
|
298
|
+
identity_status: "provisional",
|
|
299
|
+
display_label: clippedText(`${row.title} - ${row.artist}`),
|
|
300
|
+
title: row.title,
|
|
301
|
+
artist_credits: [{ name: row.artist, role: "unknown" }],
|
|
302
|
+
external_refs: [
|
|
303
|
+
{
|
|
304
|
+
system: "spotify-account-data",
|
|
305
|
+
entity_type: "spotify.account_data.track_label",
|
|
306
|
+
external_id: trackFingerprint,
|
|
307
|
+
},
|
|
308
|
+
],
|
|
309
|
+
created_at: row.occurredAt,
|
|
310
|
+
});
|
|
311
|
+
}
|
|
312
|
+
|
|
313
|
+
const eventIdentity = spotifyAccountDataEventIdentity({
|
|
314
|
+
subjectId: normalizedSubjectId,
|
|
315
|
+
occurredAt: row.occurredAt,
|
|
316
|
+
artist: row.artist,
|
|
317
|
+
title: row.title,
|
|
318
|
+
playedMs: row.playedMs,
|
|
319
|
+
duplicateOrdinal,
|
|
320
|
+
});
|
|
321
|
+
const eventFingerprint = eventIdentity.fingerprint;
|
|
322
|
+
listeningEvents.push({
|
|
323
|
+
schema_version: 1,
|
|
324
|
+
listening_event_id: eventIdentity.listening_event_id,
|
|
325
|
+
subject_id: normalizedSubjectId,
|
|
326
|
+
track_ref_id: trackRefId,
|
|
327
|
+
track_ref_revision: 1,
|
|
328
|
+
event_type: "play_observed",
|
|
329
|
+
occurred_at: row.occurredAt,
|
|
330
|
+
ingested_at: normalizedCapturedAt,
|
|
331
|
+
interaction_mode: "unknown",
|
|
332
|
+
played_ms: row.playedMs,
|
|
333
|
+
context: { context_type: "unknown" },
|
|
334
|
+
provenance: {
|
|
335
|
+
source_kind: "import",
|
|
336
|
+
captured_at: normalizedCapturedAt,
|
|
337
|
+
external_ref: {
|
|
338
|
+
system: "spotify-account-data",
|
|
339
|
+
entity_type: "spotify.account_data.play_history",
|
|
340
|
+
external_id: eventFingerprint,
|
|
341
|
+
},
|
|
342
|
+
import_batch_id: importBatchId,
|
|
343
|
+
},
|
|
344
|
+
deduplication_key: eventFingerprint,
|
|
345
|
+
});
|
|
346
|
+
}
|
|
347
|
+
|
|
348
|
+
const earliestOccurredAt = rows[0]?.occurredAt ?? null;
|
|
349
|
+
const latestOccurredAt = rows.at(-1)?.occurredAt ?? null;
|
|
350
|
+
return {
|
|
351
|
+
source_key: SPOTIFY_ACCOUNT_DATA_HISTORY_SOURCE,
|
|
352
|
+
captured_at: normalizedCapturedAt,
|
|
353
|
+
cursor_after_ms:
|
|
354
|
+
latestOccurredAt === null ? null : Date.parse(latestOccurredAt),
|
|
355
|
+
track_refs: [...trackRefs.values()],
|
|
356
|
+
listening_events: listeningEvents,
|
|
357
|
+
import_batch: {
|
|
358
|
+
import_batch_id: importBatchId,
|
|
359
|
+
subject_id: normalizedSubjectId,
|
|
360
|
+
source_format: SPOTIFY_ACCOUNT_DATA_HISTORY_FORMAT,
|
|
361
|
+
data_scope: SPOTIFY_ACCOUNT_DATA_HISTORY_SCOPE,
|
|
362
|
+
archive_sha256: archiveSha256,
|
|
363
|
+
archive_size_bytes: archiveSizeBytes,
|
|
364
|
+
member_names: normalizedMembers,
|
|
365
|
+
input_records: rows.length,
|
|
366
|
+
earliest_occurred_at: earliestOccurredAt,
|
|
367
|
+
latest_occurred_at: latestOccurredAt,
|
|
368
|
+
imported_at: normalizedCapturedAt,
|
|
369
|
+
},
|
|
370
|
+
};
|
|
371
|
+
}
|
|
372
|
+
|
|
373
|
+
async function sha256File(filePath) {
|
|
374
|
+
const hash = createHash("sha256");
|
|
375
|
+
for await (const chunk of createReadStream(filePath)) hash.update(chunk);
|
|
376
|
+
return hash.digest("hex");
|
|
377
|
+
}
|
|
378
|
+
|
|
379
|
+
async function unzip(args, maximumBytes) {
|
|
380
|
+
try {
|
|
381
|
+
const { stdout } = await execFileAsync("/usr/bin/unzip", args, {
|
|
382
|
+
encoding: "utf8",
|
|
383
|
+
maxBuffer: maximumBytes,
|
|
384
|
+
windowsHide: true,
|
|
385
|
+
});
|
|
386
|
+
return stdout;
|
|
387
|
+
} catch {
|
|
388
|
+
fail(
|
|
389
|
+
"The Spotify account-data ZIP could not be read safely.",
|
|
390
|
+
"spotify_account_data_archive_unreadable",
|
|
391
|
+
);
|
|
392
|
+
}
|
|
393
|
+
}
|
|
394
|
+
|
|
395
|
+
function selectedMusicMembers(listing) {
|
|
396
|
+
const selected = listing
|
|
397
|
+
.split(/\r?\n/u)
|
|
398
|
+
.filter(Boolean)
|
|
399
|
+
.map((name) => {
|
|
400
|
+
const match = musicMemberPattern.exec(name);
|
|
401
|
+
return match ? { name, index: Number(match[1]) } : null;
|
|
402
|
+
})
|
|
403
|
+
.filter(Boolean)
|
|
404
|
+
.sort((left, right) => left.index - right.index);
|
|
405
|
+
if (selected.length < 1) {
|
|
406
|
+
fail("The ZIP does not contain Spotify Account Data music history files.");
|
|
407
|
+
}
|
|
408
|
+
if (selected.length > SPOTIFY_ACCOUNT_DATA_HISTORY_LIMITS.maximumHistoryMembers) {
|
|
409
|
+
fail("The ZIP contains too many Spotify music history files.");
|
|
410
|
+
}
|
|
411
|
+
if (new Set(selected.map(({ name }) => name)).size !== selected.length) {
|
|
412
|
+
fail("The ZIP contains duplicate Spotify music history files.");
|
|
413
|
+
}
|
|
414
|
+
for (let index = 0; index < selected.length; index += 1) {
|
|
415
|
+
if (selected[index].index !== index) {
|
|
416
|
+
fail("The ZIP has a missing Spotify music history segment.");
|
|
417
|
+
}
|
|
418
|
+
}
|
|
419
|
+
return selected;
|
|
420
|
+
}
|
|
421
|
+
|
|
422
|
+
function selectedProfileMembers(listing) {
|
|
423
|
+
const supported = new Set(SPOTIFY_ACCOUNT_PROFILE_MEMBERS);
|
|
424
|
+
const selected = listing
|
|
425
|
+
.split(/\r?\n/u)
|
|
426
|
+
.filter(Boolean)
|
|
427
|
+
.map((name) => {
|
|
428
|
+
const prefix = "Spotify Account Data/";
|
|
429
|
+
if (!name.startsWith(prefix)) return null;
|
|
430
|
+
const basename = name.slice(prefix.length);
|
|
431
|
+
return supported.has(basename) ? { name, basename } : null;
|
|
432
|
+
})
|
|
433
|
+
.filter(Boolean)
|
|
434
|
+
.sort((left, right) => lexicalCompare(left.basename, right.basename));
|
|
435
|
+
if (new Set(selected.map(({ basename }) => basename)).size !== selected.length) {
|
|
436
|
+
fail("The ZIP contains duplicate Spotify account-profile files.");
|
|
437
|
+
}
|
|
438
|
+
return selected;
|
|
439
|
+
}
|
|
440
|
+
|
|
441
|
+
export async function readSpotifyAccountDataHistoryArchive({
|
|
442
|
+
archivePath,
|
|
443
|
+
subjectId,
|
|
444
|
+
capturedAt = new Date().toISOString(),
|
|
445
|
+
} = {}) {
|
|
446
|
+
if (typeof archivePath !== "string" || !archivePath.trim()) {
|
|
447
|
+
fail("A Spotify account-data ZIP path is required.");
|
|
448
|
+
}
|
|
449
|
+
const resolvedPath = path.resolve(archivePath);
|
|
450
|
+
if (path.extname(resolvedPath).toLowerCase() !== ".zip") {
|
|
451
|
+
fail("Spotify account data must be supplied as a ZIP archive.");
|
|
452
|
+
}
|
|
453
|
+
let file;
|
|
454
|
+
try {
|
|
455
|
+
file = await lstat(resolvedPath);
|
|
456
|
+
} catch {
|
|
457
|
+
fail(
|
|
458
|
+
"The Spotify account-data ZIP could not be opened.",
|
|
459
|
+
"spotify_account_data_archive_unreadable",
|
|
460
|
+
);
|
|
461
|
+
}
|
|
462
|
+
if (
|
|
463
|
+
!file.isFile() ||
|
|
464
|
+
file.isSymbolicLink() ||
|
|
465
|
+
file.size < 1 ||
|
|
466
|
+
file.size > SPOTIFY_ACCOUNT_DATA_HISTORY_LIMITS.maximumArchiveBytes
|
|
467
|
+
) {
|
|
468
|
+
fail("The Spotify account-data ZIP is not a supported regular file.");
|
|
469
|
+
}
|
|
470
|
+
|
|
471
|
+
const listing = await unzip(["-Z1", resolvedPath], 1024 * 1024);
|
|
472
|
+
const members = selectedMusicMembers(listing);
|
|
473
|
+
const profileMembers = selectedProfileMembers(listing);
|
|
474
|
+
const records = [];
|
|
475
|
+
const documents = {};
|
|
476
|
+
let totalJsonBytes = 0;
|
|
477
|
+
for (const member of members) {
|
|
478
|
+
const jsonText = await unzip(
|
|
479
|
+
["-p", resolvedPath, member.name],
|
|
480
|
+
SPOTIFY_ACCOUNT_DATA_HISTORY_LIMITS.maximumMemberBytes,
|
|
481
|
+
);
|
|
482
|
+
totalJsonBytes += Buffer.byteLength(jsonText, "utf8");
|
|
483
|
+
if (
|
|
484
|
+
totalJsonBytes >
|
|
485
|
+
SPOTIFY_ACCOUNT_DATA_HISTORY_LIMITS.maximumTotalJsonBytes
|
|
486
|
+
) {
|
|
487
|
+
fail("Spotify music history JSON is too large to import safely.");
|
|
488
|
+
}
|
|
489
|
+
let parsed;
|
|
490
|
+
try {
|
|
491
|
+
parsed = JSON.parse(jsonText.replace(/^\uFEFF/u, ""));
|
|
492
|
+
} catch {
|
|
493
|
+
fail(`Spotify music history segment ${member.index} is invalid JSON.`);
|
|
494
|
+
}
|
|
495
|
+
if (!Array.isArray(parsed)) {
|
|
496
|
+
fail(`Spotify music history segment ${member.index} is not an array.`);
|
|
497
|
+
}
|
|
498
|
+
records.push(...parsed);
|
|
499
|
+
if (records.length > SPOTIFY_ACCOUNT_DATA_HISTORY_LIMITS.maximumRecords) {
|
|
500
|
+
fail("Spotify music history contains too many records.");
|
|
501
|
+
}
|
|
502
|
+
}
|
|
503
|
+
|
|
504
|
+
for (const member of profileMembers) {
|
|
505
|
+
const jsonText = await unzip(
|
|
506
|
+
["-p", resolvedPath, member.name],
|
|
507
|
+
SPOTIFY_ACCOUNT_DATA_HISTORY_LIMITS.maximumMemberBytes,
|
|
508
|
+
);
|
|
509
|
+
totalJsonBytes += Buffer.byteLength(jsonText, "utf8");
|
|
510
|
+
if (
|
|
511
|
+
totalJsonBytes >
|
|
512
|
+
SPOTIFY_ACCOUNT_DATA_HISTORY_LIMITS.maximumTotalJsonBytes
|
|
513
|
+
) {
|
|
514
|
+
fail("Spotify account-profile JSON is too large to import safely.");
|
|
515
|
+
}
|
|
516
|
+
try {
|
|
517
|
+
documents[member.basename] = JSON.parse(
|
|
518
|
+
jsonText.replace(/^\uFEFF/u, ""),
|
|
519
|
+
);
|
|
520
|
+
} catch {
|
|
521
|
+
fail(`Spotify account-profile file ${member.basename} is invalid JSON.`);
|
|
522
|
+
}
|
|
523
|
+
}
|
|
524
|
+
|
|
525
|
+
const archiveSha256 = await sha256File(resolvedPath);
|
|
526
|
+
const historyBundle = projectSpotifyAccountDataHistory({
|
|
527
|
+
subjectId,
|
|
528
|
+
records,
|
|
529
|
+
archiveSha256,
|
|
530
|
+
archiveSizeBytes: file.size,
|
|
531
|
+
memberNames: members.map(({ name }) => path.posix.basename(name)),
|
|
532
|
+
capturedAt,
|
|
533
|
+
});
|
|
534
|
+
if (profileMembers.length === 0) return historyBundle;
|
|
535
|
+
return mergeSpotifyAccountProfile(
|
|
536
|
+
historyBundle,
|
|
537
|
+
projectSpotifyAccountDataProfile({
|
|
538
|
+
subjectId,
|
|
539
|
+
archiveSha256,
|
|
540
|
+
archiveSizeBytes: file.size,
|
|
541
|
+
documents,
|
|
542
|
+
capturedAt,
|
|
543
|
+
}),
|
|
544
|
+
);
|
|
545
|
+
}
|