@media-engine/providers 0.1.0 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +149 -242
- package/README.ru.md +204 -0
- package/dist/aniliberty-streaming/client.d.ts +33 -0
- package/dist/aniliberty-streaming/client.js +188 -0
- package/dist/aniliberty-streaming/client.js.map +1 -0
- package/dist/aniliberty-streaming/config.d.ts +24 -0
- package/dist/aniliberty-streaming/config.js +61 -0
- package/dist/aniliberty-streaming/config.js.map +1 -0
- package/dist/aniliberty-streaming/index.d.ts +4 -0
- package/dist/aniliberty-streaming/index.js +45 -0
- package/dist/aniliberty-streaming/index.js.map +1 -0
- package/dist/aniliberty-streaming/mapping.d.ts +3 -0
- package/dist/aniliberty-streaming/mapping.js +87 -0
- package/dist/aniliberty-streaming/mapping.js.map +1 -0
- package/dist/aniliberty-streaming/matching.d.ts +4 -0
- package/dist/aniliberty-streaming/matching.js +44 -0
- package/dist/aniliberty-streaming/matching.js.map +1 -0
- package/dist/anilist/graphql.d.ts +2 -0
- package/dist/anilist/graphql.js +135 -0
- package/dist/anilist/graphql.js.map +1 -0
- package/dist/anilist/index.js +11 -6
- package/dist/anilist/index.js.map +1 -1
- package/dist/bitsearch-torrent/client.d.ts +19 -0
- package/dist/bitsearch-torrent/client.js +181 -0
- package/dist/bitsearch-torrent/client.js.map +1 -0
- package/dist/bitsearch-torrent/config.d.ts +26 -0
- package/dist/bitsearch-torrent/config.js +74 -0
- package/dist/bitsearch-torrent/config.js.map +1 -0
- package/dist/bitsearch-torrent/index.d.ts +4 -0
- package/dist/bitsearch-torrent/index.js +35 -0
- package/dist/bitsearch-torrent/index.js.map +1 -0
- package/dist/bitsearch-torrent/mapping.d.ts +3 -0
- package/dist/bitsearch-torrent/mapping.js +53 -0
- package/dist/bitsearch-torrent/mapping.js.map +1 -0
- package/dist/bitsearch-torrent/matching.d.ts +3 -0
- package/dist/bitsearch-torrent/matching.js +13 -0
- package/dist/bitsearch-torrent/matching.js.map +1 -0
- package/dist/bitsearch-torrent/quota.d.ts +9 -0
- package/dist/bitsearch-torrent/quota.js +53 -0
- package/dist/bitsearch-torrent/quota.js.map +1 -0
- package/dist/cinemeta/index.js +53 -35
- package/dist/cinemeta/index.js.map +1 -1
- package/dist/ddbb-streaming/client.d.ts +24 -0
- package/dist/ddbb-streaming/client.js +98 -0
- package/dist/ddbb-streaming/client.js.map +1 -0
- package/dist/ddbb-streaming/config.d.ts +31 -0
- package/dist/ddbb-streaming/config.js +67 -0
- package/dist/ddbb-streaming/config.js.map +1 -0
- package/dist/ddbb-streaming/index.d.ts +4 -0
- package/dist/ddbb-streaming/index.js +53 -0
- package/dist/ddbb-streaming/index.js.map +1 -0
- package/dist/ddbb-streaming/mapping.d.ts +8 -0
- package/dist/ddbb-streaming/mapping.js +164 -0
- package/dist/ddbb-streaming/mapping.js.map +1 -0
- package/dist/ddbb-streaming/validation.d.ts +3 -0
- package/dist/ddbb-streaming/validation.js +84 -0
- package/dist/ddbb-streaming/validation.js.map +1 -0
- package/dist/experimental-streaming/index.js +55 -0
- package/dist/experimental-streaming/index.js.map +1 -1
- package/dist/flixhq-streaming/client.d.ts +12 -0
- package/dist/flixhq-streaming/client.js +103 -0
- package/dist/flixhq-streaming/client.js.map +1 -0
- package/dist/flixhq-streaming/index.d.ts +2 -2
- package/dist/flixhq-streaming/index.js +67 -106
- package/dist/flixhq-streaming/index.js.map +1 -1
- package/dist/imdb-dataset/index.d.ts +20 -5
- package/dist/imdb-dataset/index.js +83 -197
- package/dist/imdb-dataset/index.js.map +1 -1
- package/dist/imdb-dataset/memory-storage.d.ts +7 -0
- package/dist/imdb-dataset/memory-storage.js +168 -0
- package/dist/imdb-dataset/memory-storage.js.map +1 -0
- package/dist/imdb-dataset/sqlite-import.d.ts +24 -0
- package/dist/imdb-dataset/sqlite-import.js +298 -0
- package/dist/imdb-dataset/sqlite-import.js.map +1 -0
- package/dist/imdb-dataset/sqlite-runtime.d.ts +3 -0
- package/dist/imdb-dataset/sqlite-runtime.js +11 -0
- package/dist/imdb-dataset/sqlite-runtime.js.map +1 -0
- package/dist/imdb-dataset/sqlite-schema.d.ts +6 -0
- package/dist/imdb-dataset/sqlite-schema.js +88 -0
- package/dist/imdb-dataset/sqlite-schema.js.map +1 -0
- package/dist/imdb-dataset/sqlite-storage.d.ts +12 -0
- package/dist/imdb-dataset/sqlite-storage.js +208 -0
- package/dist/imdb-dataset/sqlite-storage.js.map +1 -0
- package/dist/imdb-dataset/storage.d.ts +33 -0
- package/dist/imdb-dataset/storage.js +2 -0
- package/dist/imdb-dataset/storage.js.map +1 -0
- package/dist/index.d.ts +7 -0
- package/dist/index.js +7 -0
- package/dist/index.js.map +1 -1
- package/dist/jacred-torrent/client.d.ts +24 -0
- package/dist/jacred-torrent/client.js +254 -0
- package/dist/jacred-torrent/client.js.map +1 -0
- package/dist/jacred-torrent/config.d.ts +24 -0
- package/dist/jacred-torrent/config.js +70 -0
- package/dist/jacred-torrent/config.js.map +1 -0
- package/dist/jacred-torrent/index.d.ts +4 -0
- package/dist/jacred-torrent/index.js +45 -0
- package/dist/jacred-torrent/index.js.map +1 -0
- package/dist/jacred-torrent/mapping.d.ts +3 -0
- package/dist/jacred-torrent/mapping.js +129 -0
- package/dist/jacred-torrent/mapping.js.map +1 -0
- package/dist/jacred-torrent/matching.d.ts +3 -0
- package/dist/jacred-torrent/matching.js +19 -0
- package/dist/jacred-torrent/matching.js.map +1 -0
- package/dist/kinobd/index.js +15 -33
- package/dist/kinobd/index.js.map +1 -1
- package/dist/kinobd-streaming/availability.d.ts +3 -0
- package/dist/kinobd-streaming/availability.js +179 -0
- package/dist/kinobd-streaming/availability.js.map +1 -0
- package/dist/kinobd-streaming/candidates.d.ts +32 -0
- package/dist/kinobd-streaming/candidates.js +282 -0
- package/dist/kinobd-streaming/candidates.js.map +1 -0
- package/dist/kinobd-streaming/config.d.ts +64 -0
- package/dist/kinobd-streaming/config.js +129 -0
- package/dist/kinobd-streaming/config.js.map +1 -0
- package/dist/kinobd-streaming/index.d.ts +3 -30
- package/dist/kinobd-streaming/index.js +2 -904
- package/dist/kinobd-streaming/index.js.map +1 -1
- package/dist/kinobd-streaming/players.d.ts +24 -0
- package/dist/kinobd-streaming/players.js +270 -0
- package/dist/kinobd-streaming/players.js.map +1 -0
- package/dist/kinobd-streaming/request-budget.d.ts +15 -0
- package/dist/kinobd-streaming/request-budget.js +61 -0
- package/dist/kinobd-streaming/request-budget.js.map +1 -0
- package/dist/kinobd-streaming/validation.d.ts +11 -0
- package/dist/kinobd-streaming/validation.js +170 -0
- package/dist/kinobd-streaming/validation.js.map +1 -0
- package/dist/magnetz-torrent/client.d.ts +16 -0
- package/dist/magnetz-torrent/client.js +235 -0
- package/dist/magnetz-torrent/client.js.map +1 -0
- package/dist/magnetz-torrent/config.d.ts +27 -0
- package/dist/magnetz-torrent/config.js +73 -0
- package/dist/magnetz-torrent/config.js.map +1 -0
- package/dist/magnetz-torrent/index.d.ts +4 -0
- package/dist/magnetz-torrent/index.js +35 -0
- package/dist/magnetz-torrent/index.js.map +1 -0
- package/dist/magnetz-torrent/mapping.d.ts +3 -0
- package/dist/magnetz-torrent/mapping.js +52 -0
- package/dist/magnetz-torrent/mapping.js.map +1 -0
- package/dist/magnetz-torrent/matching.d.ts +3 -0
- package/dist/magnetz-torrent/matching.js +5 -0
- package/dist/magnetz-torrent/matching.js.map +1 -0
- package/dist/magnetz-torrent/request-gate.d.ts +9 -0
- package/dist/magnetz-torrent/request-gate.js +48 -0
- package/dist/magnetz-torrent/request-gate.js.map +1 -0
- package/dist/package-version.d.ts +2 -0
- package/dist/package-version.js +5 -0
- package/dist/package-version.js.map +1 -0
- package/dist/shared/abort.d.ts +3 -0
- package/dist/shared/abort.js +30 -0
- package/dist/shared/abort.js.map +1 -0
- package/dist/shared/http.d.ts +13 -2
- package/dist/shared/http.js +77 -75
- package/dist/shared/http.js.map +1 -1
- package/dist/shared/index.d.ts +2 -0
- package/dist/shared/index.js +2 -0
- package/dist/shared/index.js.map +1 -1
- package/dist/shared/mapping.d.ts +5 -0
- package/dist/shared/mapping.js +32 -0
- package/dist/shared/mapping.js.map +1 -0
- package/dist/shared/options.d.ts +1 -0
- package/dist/shared/options.js +10 -0
- package/dist/shared/options.js.map +1 -0
- package/dist/shared/output-url.d.ts +2 -0
- package/dist/shared/output-url.js +50 -0
- package/dist/shared/output-url.js.map +1 -0
- package/dist/shared/rate-limit.d.ts +11 -0
- package/dist/shared/rate-limit.js +52 -0
- package/dist/shared/rate-limit.js.map +1 -0
- package/dist/shared/response-body.d.ts +6 -0
- package/dist/shared/response-body.js +109 -0
- package/dist/shared/response-body.js.map +1 -0
- package/dist/shared/retry.d.ts +10 -0
- package/dist/shared/retry.js +31 -0
- package/dist/shared/retry.js.map +1 -0
- package/dist/shared/safe-fetch.d.ts +14 -0
- package/dist/shared/safe-fetch.js +274 -0
- package/dist/shared/safe-fetch.js.map +1 -0
- package/dist/shared/torrent-release-matching.d.ts +7 -0
- package/dist/shared/torrent-release-matching.js +143 -0
- package/dist/shared/torrent-release-matching.js.map +1 -0
- package/dist/shared/torrent-release-metadata.d.ts +4 -0
- package/dist/shared/torrent-release-metadata.js +95 -0
- package/dist/shared/torrent-release-metadata.js.map +1 -0
- package/dist/shikimori/index.js +19 -11
- package/dist/shikimori/index.js.map +1 -1
- package/dist/tvmaze/index.d.ts +11 -0
- package/dist/tvmaze/index.js +274 -0
- package/dist/tvmaze/index.js.map +1 -0
- package/dist/wikidata/cache.d.ts +11 -0
- package/dist/wikidata/cache.js +30 -0
- package/dist/wikidata/cache.js.map +1 -0
- package/dist/wikidata/candidates.d.ts +12 -0
- package/dist/wikidata/candidates.js +67 -0
- package/dist/wikidata/candidates.js.map +1 -0
- package/dist/wikidata/client.d.ts +40 -0
- package/dist/wikidata/client.js +224 -0
- package/dist/wikidata/client.js.map +1 -0
- package/dist/wikidata/index.d.ts +3 -0
- package/dist/wikidata/index.js +34 -111
- package/dist/wikidata/index.js.map +1 -1
- package/dist/yts-torrent/client.d.ts +25 -0
- package/dist/yts-torrent/client.js +163 -0
- package/dist/yts-torrent/client.js.map +1 -0
- package/dist/yts-torrent/config.d.ts +22 -0
- package/dist/yts-torrent/config.js +57 -0
- package/dist/yts-torrent/config.js.map +1 -0
- package/dist/yts-torrent/index.d.ts +4 -0
- package/dist/yts-torrent/index.js +42 -0
- package/dist/yts-torrent/index.js.map +1 -0
- package/dist/yts-torrent/mapping.d.ts +4 -0
- package/dist/yts-torrent/mapping.js +107 -0
- package/dist/yts-torrent/mapping.js.map +1 -0
- package/package.json +23 -6
|
@@ -1,7 +1,13 @@
|
|
|
1
|
+
import { createImdbDatasetMemoryStorage, } from "./memory-storage.js";
|
|
2
|
+
import { mapGenreNames, normalizeProviderSearchText } from "../shared/mapping.js";
|
|
3
|
+
import { resolveBoundedIntegerOption } from "../shared/options.js";
|
|
4
|
+
import { normalizeProviderOutputUrl } from "../shared/output-url.js";
|
|
5
|
+
export { createImdbDatasetMemoryStorage } from "./memory-storage.js";
|
|
6
|
+
export { buildImdbDatasetSqliteIndex } from "./sqlite-import.js";
|
|
7
|
+
export { IMDB_DATASET_SQLITE_SCHEMA_VERSION, openImdbDatasetSqliteStorage, } from "./sqlite-storage.js";
|
|
1
8
|
const PROVIDER_NAME = "imdb-dataset";
|
|
2
9
|
const DEFAULT_SEARCH_LIMIT = 20;
|
|
3
|
-
|
|
4
|
-
// Создает локальный IMDb dataset provider без live scraping и неофициальных API.
|
|
10
|
+
const MAX_SEARCH_LIMIT = 100;
|
|
5
11
|
export function imdbDatasetProvider(options) {
|
|
6
12
|
const config = createImdbDatasetConfig(options);
|
|
7
13
|
return {
|
|
@@ -20,118 +26,116 @@ export function imdbDatasetProvider(options) {
|
|
|
20
26
|
features: ["ratings", "genres"],
|
|
21
27
|
},
|
|
22
28
|
async search(query, context) {
|
|
23
|
-
return searchImdbDataset(config, query, context
|
|
29
|
+
return searchImdbDataset(config, query, context);
|
|
24
30
|
},
|
|
25
31
|
async getDetails(query, context) {
|
|
26
|
-
return getImdbDatasetDetails(config, query, context
|
|
32
|
+
return getImdbDatasetDetails(config, query, context);
|
|
27
33
|
},
|
|
28
34
|
};
|
|
29
35
|
}
|
|
30
|
-
// Parses TSV content and builds lookup maps once at provider creation time.
|
|
31
|
-
// Парсит TSV content и строит lookup maps один раз при создании provider.
|
|
32
36
|
function createImdbDatasetConfig(options) {
|
|
33
|
-
const
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
37
|
+
const hasStorage = "storage" in options && options.storage !== undefined;
|
|
38
|
+
const hasTsv = "titleBasicsTsv" in options && options.titleBasicsTsv !== undefined;
|
|
39
|
+
if (hasStorage === hasTsv) {
|
|
40
|
+
throw new TypeError("IMDb dataset provider requires exactly one of storage or titleBasicsTsv");
|
|
41
|
+
}
|
|
42
|
+
if (hasStorage && !isImdbDatasetStorage(options.storage)) {
|
|
43
|
+
throw new TypeError("IMDb dataset storage must implement getTitleById and searchTitles");
|
|
44
|
+
}
|
|
45
|
+
const storage = hasStorage
|
|
46
|
+
? options.storage
|
|
47
|
+
: createImdbDatasetMemoryStorage({
|
|
48
|
+
titleBasicsTsv: options.titleBasicsTsv,
|
|
49
|
+
titleRatingsTsv: options.titleRatingsTsv,
|
|
50
|
+
includeAdult: options.includeAdult,
|
|
51
|
+
});
|
|
40
52
|
return {
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
searchLimit: options.searchLimit ?? DEFAULT_SEARCH_LIMIT,
|
|
53
|
+
storage,
|
|
54
|
+
searchLimit: resolveBoundedIntegerOption(options.searchLimit, DEFAULT_SEARCH_LIMIT, "IMDb dataset searchLimit", 1, MAX_SEARCH_LIMIT),
|
|
44
55
|
};
|
|
45
56
|
}
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
57
|
+
function isImdbDatasetStorage(value) {
|
|
58
|
+
return (typeof value === "object" &&
|
|
59
|
+
value !== null &&
|
|
60
|
+
"getTitleById" in value &&
|
|
61
|
+
typeof value.getTitleById === "function" &&
|
|
62
|
+
"searchTitles" in value &&
|
|
63
|
+
typeof value.searchTitles === "function");
|
|
64
|
+
}
|
|
65
|
+
async function searchImdbDataset(config, query, context) {
|
|
49
66
|
if (query.type === "anime") {
|
|
50
67
|
return [];
|
|
51
68
|
}
|
|
69
|
+
context.signal?.throwIfAborted();
|
|
52
70
|
if (query.ids?.imdb) {
|
|
53
|
-
const record = config.
|
|
71
|
+
const record = await config.storage.getTitleById(query.ids.imdb, {
|
|
72
|
+
signal: context.signal,
|
|
73
|
+
});
|
|
74
|
+
context.signal?.throwIfAborted();
|
|
54
75
|
return record && matchesType(record, query.type)
|
|
55
|
-
? [createSearchResult(
|
|
76
|
+
? [createSearchResult(record, context.debug, 1)]
|
|
56
77
|
: [];
|
|
57
78
|
}
|
|
58
|
-
const normalizedTitle =
|
|
59
|
-
if (!normalizedTitle) {
|
|
79
|
+
const normalizedTitle = normalizeProviderSearchText(query.title ?? "");
|
|
80
|
+
if (!normalizedTitle || query.limit === 0) {
|
|
60
81
|
return [];
|
|
61
82
|
}
|
|
62
|
-
const limit = query.limit
|
|
63
|
-
const matches =
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
}
|
|
78
|
-
matches.push(candidate);
|
|
79
|
-
matches.sort(compareSearchMatches);
|
|
80
|
-
if (matches.length > limit) {
|
|
81
|
-
matches.pop();
|
|
82
|
-
}
|
|
83
|
-
}
|
|
84
|
-
return matches.map((entry) => createSearchResult(config, entry.record, debug, entry.score));
|
|
85
|
-
}
|
|
86
|
-
// Loads local details by IMDb title ID.
|
|
87
|
-
// Загружает локальные details по IMDb title ID.
|
|
88
|
-
function getImdbDatasetDetails(config, query, debug) {
|
|
83
|
+
const limit = resolveBoundedIntegerOption(query.limit, config.searchLimit, "IMDb dataset query limit", 1, MAX_SEARCH_LIMIT);
|
|
84
|
+
const matches = await config.storage.searchTitles({
|
|
85
|
+
normalizedTitle,
|
|
86
|
+
type: query.type,
|
|
87
|
+
year: query.year,
|
|
88
|
+
limit,
|
|
89
|
+
signal: context.signal,
|
|
90
|
+
});
|
|
91
|
+
context.signal?.throwIfAborted();
|
|
92
|
+
return matches
|
|
93
|
+
.slice(0, limit)
|
|
94
|
+
.filter((match) => matchesStorageQuery(match, query))
|
|
95
|
+
.map((match) => createSearchResult(match.record, context.debug, normalizeConfidence(match.confidence)));
|
|
96
|
+
}
|
|
97
|
+
async function getImdbDatasetDetails(config, query, context) {
|
|
89
98
|
const imdbId = query.ids?.imdb;
|
|
90
99
|
if (!imdbId || query.type === "anime") {
|
|
91
100
|
return null;
|
|
92
101
|
}
|
|
93
|
-
|
|
102
|
+
context.signal?.throwIfAborted();
|
|
103
|
+
const record = await config.storage.getTitleById(imdbId, { signal: context.signal });
|
|
104
|
+
context.signal?.throwIfAborted();
|
|
94
105
|
if (!record || !matchesType(record, query.type)) {
|
|
95
106
|
return null;
|
|
96
107
|
}
|
|
97
|
-
const details = recordToDetails(config, record);
|
|
98
108
|
return {
|
|
99
109
|
provider: PROVIDER_NAME,
|
|
100
|
-
details,
|
|
110
|
+
details: recordToDetails(record),
|
|
101
111
|
source: createProviderSource(record),
|
|
102
|
-
raw: debug ? record : undefined,
|
|
112
|
+
raw: context.debug ? record : undefined,
|
|
103
113
|
confidence: 1,
|
|
104
114
|
};
|
|
105
115
|
}
|
|
106
|
-
|
|
107
|
-
// Преобразует одну IMDb title row в provider search result.
|
|
108
|
-
function createSearchResult(config, record, debug, confidence) {
|
|
116
|
+
function createSearchResult(record, debug, confidence) {
|
|
109
117
|
return {
|
|
110
118
|
provider: PROVIDER_NAME,
|
|
111
|
-
item: recordToItem(
|
|
119
|
+
item: recordToItem(record),
|
|
112
120
|
source: createProviderSource(record),
|
|
113
121
|
raw: debug ? record : undefined,
|
|
114
122
|
confidence,
|
|
115
123
|
};
|
|
116
124
|
}
|
|
117
|
-
|
|
118
|
-
// Преобразует IMDb title record в compact MediaItem.
|
|
119
|
-
function recordToItem(config, record) {
|
|
125
|
+
function recordToItem(record) {
|
|
120
126
|
return {
|
|
121
|
-
id: `${PROVIDER_NAME}-${record.
|
|
127
|
+
id: `${PROVIDER_NAME}-${record.imdbId}`,
|
|
122
128
|
type: record.type,
|
|
123
129
|
title: record.primaryTitle,
|
|
124
130
|
originalTitle: record.originalTitle,
|
|
125
131
|
year: record.startYear,
|
|
126
|
-
genres:
|
|
127
|
-
ratings: mapRating(
|
|
132
|
+
genres: mapGenreNames(record.genres ? [...record.genres] : undefined, PROVIDER_NAME),
|
|
133
|
+
ratings: mapRating(record.rating),
|
|
128
134
|
ids: createIds(record),
|
|
129
135
|
};
|
|
130
136
|
}
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
function recordToDetails(config, record) {
|
|
134
|
-
const item = recordToItem(config, record);
|
|
137
|
+
function recordToDetails(record) {
|
|
138
|
+
const item = recordToItem(record);
|
|
135
139
|
const sourceProviders = [createProviderSource(record)];
|
|
136
140
|
if (record.type === "series") {
|
|
137
141
|
const details = {
|
|
@@ -150,105 +154,27 @@ function recordToDetails(config, record) {
|
|
|
150
154
|
};
|
|
151
155
|
return details;
|
|
152
156
|
}
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
return undefined;
|
|
159
|
-
}
|
|
160
|
-
const originalTitle = emptyToUndefined(row.originalTitle);
|
|
161
|
-
return {
|
|
162
|
-
tconst: row.tconst,
|
|
163
|
-
type,
|
|
164
|
-
primaryTitle: row.primaryTitle,
|
|
165
|
-
originalTitle,
|
|
166
|
-
startYear: parseNumber(row.startYear),
|
|
167
|
-
endYear: parseNumber(row.endYear),
|
|
168
|
-
runtimeMinutes: parseNumber(row.runtimeMinutes),
|
|
169
|
-
genres: parseList(row.genres),
|
|
170
|
-
normalizedPrimaryTitle: normalizeSearchText(row.primaryTitle),
|
|
171
|
-
normalizedOriginalTitle: normalizeSearchText(originalTitle ?? ""),
|
|
172
|
-
};
|
|
173
|
-
}
|
|
174
|
-
// Parses optional IMDb title.ratings TSV content.
|
|
175
|
-
// Парсит опциональный IMDb title.ratings TSV content.
|
|
176
|
-
function parseRatings(tsv) {
|
|
177
|
-
const ratings = new Map();
|
|
178
|
-
if (!tsv) {
|
|
179
|
-
return ratings;
|
|
180
|
-
}
|
|
181
|
-
for (const row of parseTsv(tsv)) {
|
|
182
|
-
const rating = parseNumber(row.averageRating);
|
|
183
|
-
if (row.tconst && rating !== undefined) {
|
|
184
|
-
ratings.set(row.tconst, {
|
|
185
|
-
averageRating: rating,
|
|
186
|
-
numVotes: parseNumber(row.numVotes),
|
|
187
|
-
});
|
|
188
|
-
}
|
|
189
|
-
}
|
|
190
|
-
return ratings;
|
|
157
|
+
function matchesStorageQuery(match, query) {
|
|
158
|
+
return (Number.isFinite(match.confidence) &&
|
|
159
|
+
match.confidence > 0 &&
|
|
160
|
+
matchesType(match.record, query.type) &&
|
|
161
|
+
(query.year === undefined || match.record.startYear === query.year));
|
|
191
162
|
}
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
function* parseTsv(content) {
|
|
195
|
-
const firstLineEnd = content.indexOf("\n");
|
|
196
|
-
const headerLine = (firstLineEnd < 0 ? content : content.slice(0, firstLineEnd)).replace(/\r$/, "");
|
|
197
|
-
const headers = headerLine.split("\t");
|
|
198
|
-
let offset = firstLineEnd < 0 ? content.length : firstLineEnd + 1;
|
|
199
|
-
while (offset < content.length) {
|
|
200
|
-
const nextLineEnd = content.indexOf("\n", offset);
|
|
201
|
-
const lineEnd = nextLineEnd < 0 ? content.length : nextLineEnd;
|
|
202
|
-
const line = content.slice(offset, lineEnd).replace(/\r$/, "");
|
|
203
|
-
offset = nextLineEnd < 0 ? content.length : nextLineEnd + 1;
|
|
204
|
-
if (!line) {
|
|
205
|
-
continue;
|
|
206
|
-
}
|
|
207
|
-
const values = line.split("\t");
|
|
208
|
-
const row = {};
|
|
209
|
-
headers.forEach((header, index) => {
|
|
210
|
-
row[header] = values[index] ?? "";
|
|
211
|
-
});
|
|
212
|
-
yield row;
|
|
213
|
-
}
|
|
214
|
-
}
|
|
215
|
-
// Maps IMDb titleType into Media Engine type.
|
|
216
|
-
// Мапит IMDb titleType в тип Media Engine.
|
|
217
|
-
function mapTitleType(titleType) {
|
|
218
|
-
if (titleType === "movie") {
|
|
219
|
-
return "movie";
|
|
220
|
-
}
|
|
221
|
-
if (titleType === "tvSeries") {
|
|
222
|
-
return "series";
|
|
223
|
-
}
|
|
224
|
-
return undefined;
|
|
163
|
+
function matchesType(record, type) {
|
|
164
|
+
return type === undefined || type === record.type;
|
|
225
165
|
}
|
|
226
|
-
// Creates normalized IMDb external IDs.
|
|
227
|
-
// Создает нормализованные IMDb external IDs.
|
|
228
166
|
function createIds(record) {
|
|
229
167
|
return {
|
|
230
|
-
imdb: record.
|
|
168
|
+
imdb: record.imdbId,
|
|
231
169
|
};
|
|
232
170
|
}
|
|
233
|
-
// Creates source attribution for IMDb dataset records.
|
|
234
|
-
// Создает source attribution для IMDb dataset records.
|
|
235
171
|
function createProviderSource(record) {
|
|
236
172
|
return {
|
|
237
173
|
provider: PROVIDER_NAME,
|
|
238
174
|
ids: createIds(record),
|
|
239
|
-
url: `https://www.imdb.com/title/${record.
|
|
175
|
+
url: normalizeProviderOutputUrl(`https://www.imdb.com/title/${record.imdbId}/`),
|
|
240
176
|
};
|
|
241
177
|
}
|
|
242
|
-
// Maps IMDb genres into normalized genre labels.
|
|
243
|
-
// Мапит IMDb genres в нормализованные genre labels.
|
|
244
|
-
function mapGenres(genres) {
|
|
245
|
-
return genres?.map((genre) => ({
|
|
246
|
-
name: genre,
|
|
247
|
-
source: PROVIDER_NAME,
|
|
248
|
-
}));
|
|
249
|
-
}
|
|
250
|
-
// Maps IMDb rating row into normalized rating.
|
|
251
|
-
// Мапит IMDb rating row в нормализованный rating.
|
|
252
178
|
function mapRating(rating) {
|
|
253
179
|
return rating
|
|
254
180
|
? [
|
|
@@ -261,47 +187,7 @@ function mapRating(rating) {
|
|
|
261
187
|
]
|
|
262
188
|
: undefined;
|
|
263
189
|
}
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
function scoreTitle(record, normalizedQuery) {
|
|
267
|
-
const primary = record.normalizedPrimaryTitle;
|
|
268
|
-
const original = record.normalizedOriginalTitle;
|
|
269
|
-
if (primary === normalizedQuery || original === normalizedQuery) {
|
|
270
|
-
return 1;
|
|
271
|
-
}
|
|
272
|
-
if (primary.includes(normalizedQuery) || original.includes(normalizedQuery)) {
|
|
273
|
-
return 0.75;
|
|
274
|
-
}
|
|
275
|
-
return 0;
|
|
276
|
-
}
|
|
277
|
-
function matchesType(record, type) {
|
|
278
|
-
return type === undefined || type === record.type;
|
|
279
|
-
}
|
|
280
|
-
function normalizeSearchText(value) {
|
|
281
|
-
return value
|
|
282
|
-
.toLocaleLowerCase()
|
|
283
|
-
.replaceAll(/[^\p{L}\p{N}]+/gu, " ")
|
|
284
|
-
.trim();
|
|
285
|
-
}
|
|
286
|
-
function parseList(value) {
|
|
287
|
-
const normalized = emptyToUndefined(value);
|
|
288
|
-
return normalized ? normalized.split(",").filter(Boolean) : undefined;
|
|
289
|
-
}
|
|
290
|
-
function parseNumber(value) {
|
|
291
|
-
const normalized = emptyToUndefined(value);
|
|
292
|
-
if (!normalized) {
|
|
293
|
-
return undefined;
|
|
294
|
-
}
|
|
295
|
-
const parsed = Number(normalized);
|
|
296
|
-
return Number.isFinite(parsed) ? parsed : undefined;
|
|
297
|
-
}
|
|
298
|
-
function emptyToUndefined(value) {
|
|
299
|
-
return value && value !== "\\N" ? value : undefined;
|
|
300
|
-
}
|
|
301
|
-
function compareNumbers(left, right) {
|
|
302
|
-
return (right ?? 0) - (left ?? 0);
|
|
303
|
-
}
|
|
304
|
-
function compareSearchMatches(left, right) {
|
|
305
|
-
return right.score - left.score || compareNumbers(left.record.startYear, right.record.startYear);
|
|
190
|
+
function normalizeConfidence(confidence) {
|
|
191
|
+
return Math.min(1, Math.max(0, confidence));
|
|
306
192
|
}
|
|
307
193
|
//# sourceMappingURL=index.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/imdb-dataset/index.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/imdb-dataset/index.ts"],"names":[],"mappings":"AAeA,OAAO,EACL,8BAA8B,GAE/B,MAAM,qBAAqB,CAAC;AAM7B,OAAO,EAAE,aAAa,EAAE,2BAA2B,EAAE,MAAM,sBAAsB,CAAC;AAClF,OAAO,EAAE,2BAA2B,EAAE,MAAM,sBAAsB,CAAC;AACnE,OAAO,EAAE,0BAA0B,EAAE,MAAM,yBAAyB,CAAC;AAErE,OAAO,EAAE,8BAA8B,EAAE,MAAM,qBAAqB,CAAC;AAErE,OAAO,EAAE,2BAA2B,EAAE,MAAM,oBAAoB,CAAC;AAOjE,OAAO,EACL,kCAAkC,EAClC,4BAA4B,GAG7B,MAAM,qBAAqB,CAAC;AAU7B,MAAM,aAAa,GAAG,cAAc,CAAC;AACrC,MAAM,oBAAoB,GAAG,EAAE,CAAC;AAChC,MAAM,gBAAgB,GAAG,GAAG,CAAC;AA4B7B,MAAM,UAAU,mBAAmB,CAAC,OAAyC;IAC3E,MAAM,MAAM,GAAG,uBAAuB,CAAC,OAAO,CAAC,CAAC;IAEhD,OAAO;QACL,IAAI,EAAE,aAAa;QACnB,OAAO,EAAE,OAAO,CAAC,OAAO;QACxB,IAAI,EAAE,UAAU;QAChB,YAAY,EAAE;YACZ,UAAU,EAAE,CAAC,OAAO,EAAE,QAAQ,CAAC;YAC/B,MAAM,EAAE;gBACN,OAAO,EAAE,IAAI;gBACb,aAAa,EAAE,CAAC,MAAM,CAAC;aACxB;YACD,OAAO,EAAE;gBACP,aAAa,EAAE,CAAC,MAAM,CAAC;aACxB;YACD,QAAQ,EAAE,CAAC,SAAS,EAAE,QAAQ,CAAC;SAChC;QACD,KAAK,CAAC,MAAM,CAAC,KAAK,EAAE,OAAO;YACzB,OAAO,iBAAiB,CAAC,MAAM,EAAE,KAAK,EAAE,OAAO,CAAC,CAAC;QACnD,CAAC;QACD,KAAK,CAAC,UAAU,CAAC,KAAK,EAAE,OAAO;YAC7B,OAAO,qBAAqB,CAAC,MAAM,EAAE,KAAK,EAAE,OAAO,CAAC,CAAC;QACvD,CAAC;KACF,CAAC;AACJ,CAAC;AAOD,SAAS,uBAAuB,CAAC,OAAyC;IACxE,MAAM,UAAU,GAAG,SAAS,IAAI,OAAO,IAAI,OAAO,CAAC,OAAO,KAAK,SAAS,CAAC;IACzE,MAAM,MAAM,GAAG,gBAAgB,IAAI,OAAO,IAAI,OAAO,CAAC,cAAc,KAAK,SAAS,CAAC;IAEnF,IAAI,UAAU,KAAK,MAAM,EAAE,CAAC;QAC1B,MAAM,IAAI,SAAS,CAAC,yEAAyE,CAAC,CAAC;IACjG,CAAC;IAED,IAAI,UAAU,IAAI,CAAC,oBAAoB,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;QACzD,MAAM,IAAI,SAAS,CAAC,mEAAmE,CAAC,CAAC;IAC3F,CAAC;IAED,MAAM,OAAO,GAAG,UAAU;QACxB,CAAC,CAAC,OAAO,CAAC,OAAO;QACjB,CAAC,CAAC,8BAA8B,CAAC;YAC7B,cAAc,EAAE,OAAO,CAAC,cAAc;YACtC,eAAe,EAAE,OAAO,CAAC,eAAe;YACxC,YAAY,EAAE,OAAO,CAAC,YAAY;SACnC,CAAC,CAAC;IAEP,OAAO;QACL,OAAO;QACP,WAAW,EAAE,2BAA2B,CACtC,OAAO,CAAC,WAAW,EACnB,oBAAoB,EACpB,0BAA0B,EAC1B,CAAC,EACD,gBAAgB,CACjB;KACF,CAAC;AACJ,CAAC;AAED,SAAS,oBAAoB,CAAC,KAAc;IAC1C,OAAO,CACL,OAAO,KAAK,KAAK,QAAQ;QACzB,KAAK,KAAK,IAAI;QACd,cAAc,IAAI,KAAK;QACvB,OAAO,KAAK,CAAC,YAAY,KAAK,UAAU;QACxC,cAAc,IAAI,KAAK;QACvB,OAAO,KAAK,CAAC,YAAY,KAAK,UAAU,CACzC,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,iBAAiB,CAC9B,MAAyB,EACzB,KAA0B,EAC1B,OAAwB;IAExB,IAAI,KAAK,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;QAC3B,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,OAAO,CAAC,MAAM,EAAE,cAAc,EAAE,CAAC;IAEjC,IAAI,KAAK,CAAC,GAAG,EAAE,IAAI,EAAE,CAAC;QACpB,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE;YAC/D,MAAM,EAAE,OAAO,CAAC,MAAM;SACvB,CAAC,CAAC;QAEH,OAAO,CAAC,MAAM,EAAE,cAAc,EAAE,CAAC;QACjC,OAAO,MAAM,IAAI,WAAW,CAAC,MAAM,EAAE,KAAK,CAAC,IAAI,CAAC;YAC9C,CAAC,CAAC,CAAC,kBAAkB,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;YAChD,CAAC,CAAC,EAAE,CAAC;IACT,CAAC;IAED,MAAM,eAAe,GAAG,2BAA2B,CAAC,KAAK,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC;IAEvE,IAAI,CAAC,eAAe,IAAI,KAAK,CAAC,KAAK,KAAK,CAAC,EAAE,CAAC;QAC1C,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,MAAM,KAAK,GAAG,2BAA2B,CACvC,KAAK,CAAC,KAAK,EACX,MAAM,CAAC,WAAW,EAClB,0BAA0B,EAC1B,CAAC,EACD,gBAAgB,CACjB,CAAC;IACF,MAAM,OAAO,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC,YAAY,CAAC;QAChD,eAAe;QACf,IAAI,EAAE,KAAK,CAAC,IAAI;QAChB,IAAI,EAAE,KAAK,CAAC,IAAI;QAChB,KAAK;QACL,MAAM,EAAE,OAAO,CAAC,MAAM;KACvB,CAAC,CAAC;IAEH,OAAO,CAAC,MAAM,EAAE,cAAc,EAAE,CAAC;IAEjC,OAAO,OAAO;SACX,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC;SACf,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,mBAAmB,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;SACpD,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CACb,kBAAkB,CAAC,KAAK,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,EAAE,mBAAmB,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,CACvF,CAAC;AACN,CAAC;AAED,KAAK,UAAU,qBAAqB,CAClC,MAAyB,EACzB,KAA2B,EAC3B,OAAwB;IAExB,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,EAAE,IAAI,CAAC;IAE/B,IAAI,CAAC,MAAM,IAAI,KAAK,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;QACtC,OAAO,IAAI,CAAC;IACd,CAAC;IAED,OAAO,CAAC,MAAM,EAAE,cAAc,EAAE,CAAC;IACjC,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC,YAAY,CAAC,MAAM,EAAE,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;IACrF,OAAO,CAAC,MAAM,EAAE,cAAc,EAAE,CAAC;IAEjC,IAAI,CAAC,MAAM,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;QAChD,OAAO,IAAI,CAAC;IACd,CAAC;IAED,OAAO;QACL,QAAQ,EAAE,aAAa;QACvB,OAAO,EAAE,eAAe,CAAC,MAAM,CAAC;QAChC,MAAM,EAAE,oBAAoB,CAAC,MAAM,CAAC;QACpC,GAAG,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS;QACvC,UAAU,EAAE,CAAC;KACd,CAAC;AACJ,CAAC;AAED,SAAS,kBAAkB,CACzB,MAA8B,EAC9B,KAA0B,EAC1B,UAAkB;IAElB,OAAO;QACL,QAAQ,EAAE,aAAa;QACvB,IAAI,EAAE,YAAY,CAAC,MAAM,CAAC;QAC1B,MAAM,EAAE,oBAAoB,CAAC,MAAM,CAAC;QACpC,GAAG,EAAE,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS;QAC/B,UAAU;KACX,CAAC;AACJ,CAAC;AAED,SAAS,YAAY,CAAC,MAA8B;IAClD,OAAO;QACL,EAAE,EAAE,GAAG,aAAa,IAAI,MAAM,CAAC,MAAM,EAAE;QACvC,IAAI,EAAE,MAAM,CAAC,IAAI;QACjB,KAAK,EAAE,MAAM,CAAC,YAAY;QAC1B,aAAa,EAAE,MAAM,CAAC,aAAa;QACnC,IAAI,EAAE,MAAM,CAAC,SAAS;QACtB,MAAM,EAAE,aAAa,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,SAAS,EAAE,aAAa,CAAC;QACpF,OAAO,EAAE,SAAS,CAAC,MAAM,CAAC,MAAM,CAAC;QACjC,GAAG,EAAE,SAAS,CAAC,MAAM,CAAC;KACvB,CAAC;AACJ,CAAC;AAED,SAAS,eAAe,CAAC,MAA8B;IACrD,MAAM,IAAI,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC;IAClC,MAAM,eAAe,GAAG,CAAC,oBAAoB,CAAC,MAAM,CAAC,CAAC,CAAC;IAEvD,IAAI,MAAM,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;QAC7B,MAAM,OAAO,GAAkB;YAC7B,GAAG,IAAI;YACP,IAAI,EAAE,QAAQ;YACd,cAAc,EAAE,MAAM,CAAC,cAAc;YACrC,eAAe;SAChB,CAAC;QAEF,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,MAAM,OAAO,GAAiB;QAC5B,GAAG,IAAI;QACP,IAAI,EAAE,OAAO;QACb,cAAc,EAAE,MAAM,CAAC,cAAc;QACrC,eAAe;KAChB,CAAC;IAEF,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,SAAS,mBAAmB,CAC1B,KAAqC,EACrC,KAA0B;IAE1B,OAAO,CACL,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,UAAU,CAAC;QACjC,KAAK,CAAC,UAAU,GAAG,CAAC;QACpB,WAAW,CAAC,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,IAAI,CAAC;QACrC,CAAC,KAAK,CAAC,IAAI,KAAK,SAAS,IAAI,KAAK,CAAC,MAAM,CAAC,SAAS,KAAK,KAAK,CAAC,IAAI,CAAC,CACpE,CAAC;AACJ,CAAC;AAED,SAAS,WAAW,CAAC,MAA8B,EAAE,IAAiC;IACpF,OAAO,IAAI,KAAK,SAAS,IAAI,IAAI,KAAK,MAAM,CAAC,IAAI,CAAC;AACpD,CAAC;AAED,SAAS,SAAS,CAAC,MAA8B;IAC/C,OAAO;QACL,IAAI,EAAE,MAAM,CAAC,MAAM;KACpB,CAAC;AACJ,CAAC;AAED,SAAS,oBAAoB,CAAC,MAA8B;IAC1D,OAAO;QACL,QAAQ,EAAE,aAAa;QACvB,GAAG,EAAE,SAAS,CAAC,MAAM,CAAC;QACtB,GAAG,EAAE,0BAA0B,CAAC,8BAA8B,MAAM,CAAC,MAAM,GAAG,CAAC;KAChF,CAAC;AACJ,CAAC;AAED,SAAS,SAAS,CAAC,MAAwC;IACzD,OAAO,MAAM;QACX,CAAC,CAAC;YACE;gBACE,MAAM,EAAE,MAAM;gBACd,KAAK,EAAE,MAAM,CAAC,aAAa;gBAC3B,GAAG,EAAE,EAAE;gBACP,KAAK,EAAE,MAAM,CAAC,QAAQ;aACvB;SACF;QACH,CAAC,CAAC,SAAS,CAAC;AAChB,CAAC;AAED,SAAS,mBAAmB,CAAC,UAAkB;IAC7C,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC,CAAC;AAC9C,CAAC"}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import type { ImdbDatasetStorage } from "./storage.js";
|
|
2
|
+
export interface ImdbDatasetMemoryStorageOptions {
|
|
3
|
+
titleBasicsTsv: string;
|
|
4
|
+
titleRatingsTsv?: string;
|
|
5
|
+
includeAdult?: boolean;
|
|
6
|
+
}
|
|
7
|
+
export declare function createImdbDatasetMemoryStorage(options: ImdbDatasetMemoryStorageOptions): ImdbDatasetStorage;
|
|
@@ -0,0 +1,168 @@
|
|
|
1
|
+
import { normalizeProviderSearchText } from "../shared/mapping.js";
|
|
2
|
+
// Builds the legacy in-memory adapter used for small fixtures and compatibility.
|
|
3
|
+
// Создаёт прежний in-memory adapter для малых fixtures и совместимости.
|
|
4
|
+
export function createImdbDatasetMemoryStorage(options) {
|
|
5
|
+
const recordsById = new Map();
|
|
6
|
+
const ratingsById = parseRatings(options.titleRatingsTsv);
|
|
7
|
+
for (const row of parseTsv(options.titleBasicsTsv)) {
|
|
8
|
+
const record = mapTitleBasicsRow(row, options.includeAdult ?? false);
|
|
9
|
+
if (record) {
|
|
10
|
+
recordsById.set(record.imdbId, record);
|
|
11
|
+
}
|
|
12
|
+
}
|
|
13
|
+
return {
|
|
14
|
+
getTitleById(imdbId, lookupOptions) {
|
|
15
|
+
lookupOptions?.signal?.throwIfAborted();
|
|
16
|
+
const record = recordsById.get(imdbId);
|
|
17
|
+
return record ? createStorageRecord(record, ratingsById.get(imdbId)) : undefined;
|
|
18
|
+
},
|
|
19
|
+
searchTitles(query) {
|
|
20
|
+
return searchMemoryStorage(recordsById, ratingsById, query);
|
|
21
|
+
},
|
|
22
|
+
};
|
|
23
|
+
}
|
|
24
|
+
function searchMemoryStorage(recordsById, ratingsById, query) {
|
|
25
|
+
query.signal?.throwIfAborted();
|
|
26
|
+
const matches = [];
|
|
27
|
+
let visited = 0;
|
|
28
|
+
for (const record of recordsById.values()) {
|
|
29
|
+
if (visited % 4096 === 0) {
|
|
30
|
+
query.signal?.throwIfAborted();
|
|
31
|
+
}
|
|
32
|
+
visited += 1;
|
|
33
|
+
if ((query.type !== undefined && record.type !== query.type) ||
|
|
34
|
+
(query.year !== undefined && record.startYear !== query.year)) {
|
|
35
|
+
continue;
|
|
36
|
+
}
|
|
37
|
+
const confidence = scoreTitle(record, query.normalizedTitle);
|
|
38
|
+
if (confidence <= 0) {
|
|
39
|
+
continue;
|
|
40
|
+
}
|
|
41
|
+
const candidate = { record, confidence };
|
|
42
|
+
const worstMatch = matches.at(-1);
|
|
43
|
+
if (matches.length >= query.limit &&
|
|
44
|
+
worstMatch &&
|
|
45
|
+
compareSearchMatches(candidate, worstMatch) >= 0) {
|
|
46
|
+
continue;
|
|
47
|
+
}
|
|
48
|
+
matches.push(candidate);
|
|
49
|
+
matches.sort(compareSearchMatches);
|
|
50
|
+
if (matches.length > query.limit) {
|
|
51
|
+
matches.pop();
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
query.signal?.throwIfAborted();
|
|
55
|
+
return matches.map(({ record, confidence }) => ({
|
|
56
|
+
record: createStorageRecord(record, ratingsById.get(record.imdbId)),
|
|
57
|
+
confidence,
|
|
58
|
+
}));
|
|
59
|
+
}
|
|
60
|
+
function mapTitleBasicsRow(row, includeAdult) {
|
|
61
|
+
const type = mapTitleType(row.titleType);
|
|
62
|
+
if (!type || (!includeAdult && row.isAdult === "1")) {
|
|
63
|
+
return undefined;
|
|
64
|
+
}
|
|
65
|
+
const originalTitle = emptyToUndefined(row.originalTitle);
|
|
66
|
+
return {
|
|
67
|
+
imdbId: row.tconst,
|
|
68
|
+
type,
|
|
69
|
+
primaryTitle: row.primaryTitle,
|
|
70
|
+
originalTitle,
|
|
71
|
+
startYear: parseNumber(row.startYear),
|
|
72
|
+
endYear: parseNumber(row.endYear),
|
|
73
|
+
runtimeMinutes: parseNumber(row.runtimeMinutes),
|
|
74
|
+
genres: parseList(row.genres),
|
|
75
|
+
normalizedPrimaryTitle: normalizeProviderSearchText(row.primaryTitle),
|
|
76
|
+
normalizedOriginalTitle: normalizeProviderSearchText(originalTitle ?? ""),
|
|
77
|
+
};
|
|
78
|
+
}
|
|
79
|
+
function createStorageRecord(record, rating) {
|
|
80
|
+
return {
|
|
81
|
+
imdbId: record.imdbId,
|
|
82
|
+
type: record.type,
|
|
83
|
+
primaryTitle: record.primaryTitle,
|
|
84
|
+
originalTitle: record.originalTitle,
|
|
85
|
+
startYear: record.startYear,
|
|
86
|
+
endYear: record.endYear,
|
|
87
|
+
runtimeMinutes: record.runtimeMinutes,
|
|
88
|
+
genres: record.genres,
|
|
89
|
+
rating,
|
|
90
|
+
};
|
|
91
|
+
}
|
|
92
|
+
function parseRatings(tsv) {
|
|
93
|
+
const ratings = new Map();
|
|
94
|
+
if (!tsv) {
|
|
95
|
+
return ratings;
|
|
96
|
+
}
|
|
97
|
+
for (const row of parseTsv(tsv)) {
|
|
98
|
+
const rating = parseNumber(row.averageRating);
|
|
99
|
+
if (row.tconst && rating !== undefined) {
|
|
100
|
+
ratings.set(row.tconst, {
|
|
101
|
+
averageRating: rating,
|
|
102
|
+
numVotes: parseNumber(row.numVotes),
|
|
103
|
+
});
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
return ratings;
|
|
107
|
+
}
|
|
108
|
+
function* parseTsv(content) {
|
|
109
|
+
const firstLineEnd = content.indexOf("\n");
|
|
110
|
+
const headerLine = (firstLineEnd < 0 ? content : content.slice(0, firstLineEnd)).replace(/\r$/, "");
|
|
111
|
+
const headers = headerLine.split("\t");
|
|
112
|
+
let offset = firstLineEnd < 0 ? content.length : firstLineEnd + 1;
|
|
113
|
+
while (offset < content.length) {
|
|
114
|
+
const nextLineEnd = content.indexOf("\n", offset);
|
|
115
|
+
const lineEnd = nextLineEnd < 0 ? content.length : nextLineEnd;
|
|
116
|
+
const line = content.slice(offset, lineEnd).replace(/\r$/, "");
|
|
117
|
+
offset = nextLineEnd < 0 ? content.length : nextLineEnd + 1;
|
|
118
|
+
if (!line) {
|
|
119
|
+
continue;
|
|
120
|
+
}
|
|
121
|
+
const values = line.split("\t");
|
|
122
|
+
const row = {};
|
|
123
|
+
headers.forEach((header, index) => {
|
|
124
|
+
row[header] = values[index] ?? "";
|
|
125
|
+
});
|
|
126
|
+
yield row;
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
function scoreTitle(record, normalizedQuery) {
|
|
130
|
+
const primary = record.normalizedPrimaryTitle;
|
|
131
|
+
const original = record.normalizedOriginalTitle;
|
|
132
|
+
if (primary === normalizedQuery || original === normalizedQuery) {
|
|
133
|
+
return 1;
|
|
134
|
+
}
|
|
135
|
+
if (primary.includes(normalizedQuery) || original.includes(normalizedQuery)) {
|
|
136
|
+
return 0.75;
|
|
137
|
+
}
|
|
138
|
+
return 0;
|
|
139
|
+
}
|
|
140
|
+
function mapTitleType(titleType) {
|
|
141
|
+
if (titleType === "movie") {
|
|
142
|
+
return "movie";
|
|
143
|
+
}
|
|
144
|
+
if (titleType === "tvSeries") {
|
|
145
|
+
return "series";
|
|
146
|
+
}
|
|
147
|
+
return undefined;
|
|
148
|
+
}
|
|
149
|
+
function parseList(value) {
|
|
150
|
+
const normalized = emptyToUndefined(value);
|
|
151
|
+
return normalized ? normalized.split(",").filter(Boolean) : undefined;
|
|
152
|
+
}
|
|
153
|
+
function parseNumber(value) {
|
|
154
|
+
const normalized = emptyToUndefined(value);
|
|
155
|
+
if (!normalized) {
|
|
156
|
+
return undefined;
|
|
157
|
+
}
|
|
158
|
+
const parsed = Number(normalized);
|
|
159
|
+
return Number.isFinite(parsed) ? parsed : undefined;
|
|
160
|
+
}
|
|
161
|
+
function emptyToUndefined(value) {
|
|
162
|
+
return value && value !== "\\N" ? value : undefined;
|
|
163
|
+
}
|
|
164
|
+
function compareSearchMatches(left, right) {
|
|
165
|
+
return (right.confidence - left.confidence ||
|
|
166
|
+
(right.record.startYear ?? 0) - (left.record.startYear ?? 0));
|
|
167
|
+
}
|
|
168
|
+
//# sourceMappingURL=memory-storage.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"memory-storage.js","sourceRoot":"","sources":["../../src/imdb-dataset/memory-storage.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,2BAA2B,EAAE,MAAM,sBAAsB,CAAC;AAsBnE,iFAAiF;AACjF,wEAAwE;AACxE,MAAM,UAAU,8BAA8B,CAC5C,OAAwC;IAExC,MAAM,WAAW,GAAG,IAAI,GAAG,EAA8B,CAAC;IAC1D,MAAM,WAAW,GAAG,YAAY,CAAC,OAAO,CAAC,eAAe,CAAC,CAAC;IAE1D,KAAK,MAAM,GAAG,IAAI,QAAQ,CAAC,OAAO,CAAC,cAAc,CAAC,EAAE,CAAC;QACnD,MAAM,MAAM,GAAG,iBAAiB,CAAC,GAAG,EAAE,OAAO,CAAC,YAAY,IAAI,KAAK,CAAC,CAAC;QAErE,IAAI,MAAM,EAAE,CAAC;YACX,WAAW,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QACzC,CAAC;IACH,CAAC;IAED,OAAO;QACL,YAAY,CAAC,MAAM,EAAE,aAAa;YAChC,aAAa,EAAE,MAAM,EAAE,cAAc,EAAE,CAAC;YACxC,MAAM,MAAM,GAAG,WAAW,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YACvC,OAAO,MAAM,CAAC,CAAC,CAAC,mBAAmB,CAAC,MAAM,EAAE,WAAW,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;QACnF,CAAC;QACD,YAAY,CAAC,KAAK;YAChB,OAAO,mBAAmB,CAAC,WAAW,EAAE,WAAW,EAAE,KAAK,CAAC,CAAC;QAC9D,CAAC;KACF,CAAC;AACJ,CAAC;AAED,SAAS,mBAAmB,CAC1B,WAAoD,EACpD,WAAyD,EACzD,KAAoC;IAEpC,KAAK,CAAC,MAAM,EAAE,cAAc,EAAE,CAAC;IAC/B,MAAM,OAAO,GAA8D,EAAE,CAAC;IAC9E,IAAI,OAAO,GAAG,CAAC,CAAC;IAEhB,KAAK,MAAM,MAAM,IAAI,WAAW,CAAC,MAAM,EAAE,EAAE,CAAC;QAC1C,IAAI,OAAO,GAAG,IAAI,KAAK,CAAC,EAAE,CAAC;YACzB,KAAK,CAAC,MAAM,EAAE,cAAc,EAAE,CAAC;QACjC,CAAC;QACD,OAAO,IAAI,CAAC,CAAC;QAEb,IACE,CAAC,KAAK,CAAC,IAAI,KAAK,SAAS,IAAI,MAAM,CAAC,IAAI,KAAK,KAAK,CAAC,IAAI,CAAC;YACxD,CAAC,KAAK,CAAC,IAAI,KAAK,SAAS,IAAI,MAAM,CAAC,SAAS,KAAK,KAAK,CAAC,IAAI,CAAC,EAC7D,CAAC;YACD,SAAS;QACX,CAAC;QAED,MAAM,UAAU,GAAG,UAAU,CAAC,MAAM,EAAE,KAAK,CAAC,eAAe,CAAC,CAAC;QAE7D,IAAI,UAAU,IAAI,CAAC,EAAE,CAAC;YACpB,SAAS;QACX,CAAC;QAED,MAAM,SAAS,GAAG,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC;QACzC,MAAM,UAAU,GAAG,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QAElC,IACE,OAAO,CAAC,MAAM,IAAI,KAAK,CAAC,KAAK;YAC7B,UAAU;YACV,oBAAoB,CAAC,SAAS,EAAE,UAAU,CAAC,IAAI,CAAC,EAChD,CAAC;YACD,SAAS;QACX,CAAC;QAED,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QACxB,OAAO,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;QAEnC,IAAI,OAAO,CAAC,MAAM,GAAG,KAAK,CAAC,KAAK,EAAE,CAAC;YACjC,OAAO,CAAC,GAAG,EAAE,CAAC;QAChB,CAAC;IACH,CAAC;IAED,KAAK,CAAC,MAAM,EAAE,cAAc,EAAE,CAAC;IAC/B,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,EAAE,UAAU,EAAE,EAAE,EAAE,CAAC,CAAC;QAC9C,MAAM,EAAE,mBAAmB,CAAC,MAAM,EAAE,WAAW,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QACnE,UAAU;KACX,CAAC,CAAC,CAAC;AACN,CAAC;AAED,SAAS,iBAAiB,CACxB,GAA2B,EAC3B,YAAqB;IAErB,MAAM,IAAI,GAAG,YAAY,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;IAEzC,IAAI,CAAC,IAAI,IAAI,CAAC,CAAC,YAAY,IAAI,GAAG,CAAC,OAAO,KAAK,GAAG,CAAC,EAAE,CAAC;QACpD,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,MAAM,aAAa,GAAG,gBAAgB,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC;IAE1D,OAAO;QACL,MAAM,EAAE,GAAG,CAAC,MAAM;QAClB,IAAI;QACJ,YAAY,EAAE,GAAG,CAAC,YAAY;QAC9B,aAAa;QACb,SAAS,EAAE,WAAW,CAAC,GAAG,CAAC,SAAS,CAAC;QACrC,OAAO,EAAE,WAAW,CAAC,GAAG,CAAC,OAAO,CAAC;QACjC,cAAc,EAAE,WAAW,CAAC,GAAG,CAAC,cAAc,CAAC;QAC/C,MAAM,EAAE,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC;QAC7B,sBAAsB,EAAE,2BAA2B,CAAC,GAAG,CAAC,YAAY,CAAC;QACrE,uBAAuB,EAAE,2BAA2B,CAAC,aAAa,IAAI,EAAE,CAAC;KAC1E,CAAC;AACJ,CAAC;AAED,SAAS,mBAAmB,CAC1B,MAA0B,EAC1B,MAA2C;IAE3C,OAAO;QACL,MAAM,EAAE,MAAM,CAAC,MAAM;QACrB,IAAI,EAAE,MAAM,CAAC,IAAI;QACjB,YAAY,EAAE,MAAM,CAAC,YAAY;QACjC,aAAa,EAAE,MAAM,CAAC,aAAa;QACnC,SAAS,EAAE,MAAM,CAAC,SAAS;QAC3B,OAAO,EAAE,MAAM,CAAC,OAAO;QACvB,cAAc,EAAE,MAAM,CAAC,cAAc;QACrC,MAAM,EAAE,MAAM,CAAC,MAAM;QACrB,MAAM;KACP,CAAC;AACJ,CAAC;AAED,SAAS,YAAY,CAAC,GAAuB;IAC3C,MAAM,OAAO,GAAG,IAAI,GAAG,EAAmC,CAAC;IAE3D,IAAI,CAAC,GAAG,EAAE,CAAC;QACT,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,KAAK,MAAM,GAAG,IAAI,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;QAChC,MAAM,MAAM,GAAG,WAAW,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC;QAE9C,IAAI,GAAG,CAAC,MAAM,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;YACvC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,EAAE;gBACtB,aAAa,EAAE,MAAM;gBACrB,QAAQ,EAAE,WAAW,CAAC,GAAG,CAAC,QAAQ,CAAC;aACpC,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,QAAQ,CAAC,CAAC,QAAQ,CAAC,OAAe;IAChC,MAAM,YAAY,GAAG,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IAC3C,MAAM,UAAU,GAAG,CAAC,YAAY,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,YAAY,CAAC,CAAC,CAAC,OAAO,CACtF,KAAK,EACL,EAAE,CACH,CAAC;IACF,MAAM,OAAO,GAAG,UAAU,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IACvC,IAAI,MAAM,GAAG,YAAY,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,YAAY,GAAG,CAAC,CAAC;IAElE,OAAO,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;QAC/B,MAAM,WAAW,GAAG,OAAO,CAAC,OAAO,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QAClD,MAAM,OAAO,GAAG,WAAW,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,WAAW,CAAC;QAC/D,MAAM,IAAI,GAAG,OAAO,CAAC,KAAK,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;QAC/D,MAAM,GAAG,WAAW,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,WAAW,GAAG,CAAC,CAAC;QAE5D,IAAI,CAAC,IAAI,EAAE,CAAC;YACV,SAAS;QACX,CAAC;QAED,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAChC,MAAM,GAAG,GAA2B,EAAE,CAAC;QAEvC,OAAO,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE;YAChC,GAAG,CAAC,MAAM,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;QACpC,CAAC,CAAC,CAAC;QAEH,MAAM,GAAG,CAAC;IACZ,CAAC;AACH,CAAC;AAED,SAAS,UAAU,CAAC,MAA0B,EAAE,eAAuB;IACrE,MAAM,OAAO,GAAG,MAAM,CAAC,sBAAsB,CAAC;IAC9C,MAAM,QAAQ,GAAG,MAAM,CAAC,uBAAuB,CAAC;IAEhD,IAAI,OAAO,KAAK,eAAe,IAAI,QAAQ,KAAK,eAAe,EAAE,CAAC;QAChE,OAAO,CAAC,CAAC;IACX,CAAC;IAED,IAAI,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,eAAe,CAAC,EAAE,CAAC;QAC5E,OAAO,IAAI,CAAC;IACd,CAAC;IAED,OAAO,CAAC,CAAC;AACX,CAAC;AAED,SAAS,YAAY,CAAC,SAA6B;IACjD,IAAI,SAAS,KAAK,OAAO,EAAE,CAAC;QAC1B,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,IAAI,SAAS,KAAK,UAAU,EAAE,CAAC;QAC7B,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,SAAS,SAAS,CAAC,KAAyB;IAC1C,MAAM,UAAU,GAAG,gBAAgB,CAAC,KAAK,CAAC,CAAC;IAC3C,OAAO,UAAU,CAAC,CAAC,CAAC,UAAU,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;AACxE,CAAC;AAED,SAAS,WAAW,CAAC,KAAyB;IAC5C,MAAM,UAAU,GAAG,gBAAgB,CAAC,KAAK,CAAC,CAAC;IAE3C,IAAI,CAAC,UAAU,EAAE,CAAC;QAChB,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,MAAM,MAAM,GAAG,MAAM,CAAC,UAAU,CAAC,CAAC;IAClC,OAAO,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC;AACtD,CAAC;AAED,SAAS,gBAAgB,CAAC,KAAyB;IACjD,OAAO,KAAK,IAAI,KAAK,KAAK,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC;AACtD,CAAC;AAED,SAAS,oBAAoB,CAC3B,IAAwD,EACxD,KAAyD;IAEzD,OAAO,CACL,KAAK,CAAC,UAAU,GAAG,IAAI,CAAC,UAAU;QAClC,CAAC,KAAK,CAAC,MAAM,CAAC,SAAS,IAAI,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,IAAI,CAAC,CAAC,CAC7D,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
export type ImdbDatasetSqliteBuildPhase = "titles" | "ratings" | "finalize";
|
|
2
|
+
export interface ImdbDatasetSqliteBuildProgress {
|
|
3
|
+
phase: ImdbDatasetSqliteBuildPhase;
|
|
4
|
+
processedRows: number;
|
|
5
|
+
importedRows: number;
|
|
6
|
+
}
|
|
7
|
+
export interface ImdbDatasetSqliteBuildOptions {
|
|
8
|
+
titleBasicsPath: string;
|
|
9
|
+
titleRatingsPath?: string;
|
|
10
|
+
outputPath: string;
|
|
11
|
+
includeAdult?: boolean;
|
|
12
|
+
signal?: AbortSignal;
|
|
13
|
+
onProgress?: (progress: ImdbDatasetSqliteBuildProgress) => void;
|
|
14
|
+
}
|
|
15
|
+
export interface ImdbDatasetSqliteBuildResult {
|
|
16
|
+
outputPath: string;
|
|
17
|
+
schemaVersion: number;
|
|
18
|
+
titleRows: number;
|
|
19
|
+
ratingRows: number;
|
|
20
|
+
skippedTitleRows: number;
|
|
21
|
+
indexBytes: number;
|
|
22
|
+
durationMs: number;
|
|
23
|
+
}
|
|
24
|
+
export declare function buildImdbDatasetSqliteIndex(options: ImdbDatasetSqliteBuildOptions): Promise<ImdbDatasetSqliteBuildResult>;
|