node-csfd-api 3.0.0-next.20 → 3.0.0-next.22
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +41 -3
- package/esm/fetchers/index.js +4 -13
- package/esm/helpers/cinema.helper.js +22 -10
- package/esm/helpers/creator.helper.js +12 -11
- package/esm/helpers/search.helper.js +5 -3
- package/esm/helpers/user-ratings.helper.js +2 -1
- package/esm/index.js +10 -29
- package/esm/services/cinema.service.js +11 -22
- package/esm/services/creator.service.js +8 -19
- package/esm/services/movie.service.js +10 -21
- package/esm/services/search.service.js +11 -22
- package/esm/services/user-ratings.service.js +25 -36
- package/{types → esm/types}/helpers/cinema.helper.d.ts +2 -3
- package/{types → esm/types}/helpers/search.helper.d.ts +1 -1
- package/{types → esm/types}/index.d.ts +1 -1
- package/fetchers/fetch.polyfill.d.ts +1 -0
- package/fetchers/index.d.ts +1 -0
- package/{cjs/fetchers → fetchers}/index.js +4 -13
- package/helpers/cinema.helper.d.ts +18 -0
- package/{cjs/helpers → helpers}/cinema.helper.js +23 -12
- package/helpers/creator.helper.d.ts +17 -0
- package/{cjs/helpers → helpers}/creator.helper.js +12 -11
- package/helpers/global.helper.d.ts +17 -0
- package/helpers/movie.helper.d.ts +25 -0
- package/helpers/search-user.helper.d.ts +5 -0
- package/helpers/search.helper.d.ts +11 -0
- package/{cjs/helpers → helpers}/search.helper.js +5 -3
- package/helpers/user-ratings.helper.d.ts +13 -0
- package/{cjs/helpers → helpers}/user-ratings.helper.js +2 -1
- package/index.d.ts +24 -0
- package/index.js +39 -0
- package/interfaces/cinema.interface.d.ts +23 -0
- package/interfaces/creator.interface.d.ts +12 -0
- package/interfaces/global.d.ts +22 -0
- package/interfaces/movie.interface.d.ts +66 -0
- package/interfaces/search.interface.d.ts +27 -0
- package/interfaces/user-ratings.interface.d.ts +18 -0
- package/package.json +20 -14
- package/services/cinema.service.d.ts +6 -0
- package/services/cinema.service.js +34 -0
- package/services/creator.service.d.ts +6 -0
- package/services/creator.service.js +32 -0
- package/services/movie.service.d.ts +6 -0
- package/{cjs/services → services}/movie.service.js +10 -21
- package/services/search.service.d.ts +5 -0
- package/{cjs/services → services}/search.service.js +11 -22
- package/services/user-ratings.service.d.ts +7 -0
- package/{cjs/services → services}/user-ratings.service.js +25 -36
- package/vars.d.ts +6 -0
- package/cjs/index.js +0 -58
- package/cjs/services/cinema.service.js +0 -45
- package/cjs/services/creator.service.js +0 -43
- package/types/index.ts +0 -26
- /package/{types → esm/types}/fetchers/fetch.polyfill.d.ts +0 -0
- /package/{types → esm/types}/fetchers/index.d.ts +0 -0
- /package/{types → esm/types}/helpers/creator.helper.d.ts +0 -0
- /package/{types → esm/types}/helpers/global.helper.d.ts +0 -0
- /package/{types → esm/types}/helpers/movie.helper.d.ts +0 -0
- /package/{types → esm/types}/helpers/search-user.helper.d.ts +0 -0
- /package/{types → esm/types}/helpers/user-ratings.helper.d.ts +0 -0
- /package/{types → esm/types}/interfaces/cinema.interface.d.ts +0 -0
- /package/{types → esm/types}/interfaces/creator.interface.d.ts +0 -0
- /package/{types → esm/types}/interfaces/global.d.ts +0 -0
- /package/{types → esm/types}/interfaces/movie.interface.d.ts +0 -0
- /package/{types → esm/types}/interfaces/search.interface.d.ts +0 -0
- /package/{types → esm/types}/interfaces/user-ratings.interface.d.ts +0 -0
- /package/{types → esm/types}/services/cinema.service.d.ts +0 -0
- /package/{types → esm/types}/services/creator.service.d.ts +0 -0
- /package/{types → esm/types}/services/movie.service.d.ts +0 -0
- /package/{types → esm/types}/services/search.service.d.ts +0 -0
- /package/{types → esm/types}/services/user-ratings.service.d.ts +0 -0
- /package/{types → esm/types}/vars.d.ts +0 -0
- /package/{cjs/fetchers → fetchers}/fetch.polyfill.js +0 -0
- /package/{cjs/helpers → helpers}/global.helper.js +0 -0
- /package/{cjs/helpers → helpers}/movie.helper.js +0 -0
- /package/{cjs/helpers → helpers}/search-user.helper.js +0 -0
- /package/{cjs/interfaces → interfaces}/cinema.interface.js +0 -0
- /package/{cjs/interfaces → interfaces}/creator.interface.js +0 -0
- /package/{cjs/interfaces → interfaces}/global.js +0 -0
- /package/{cjs/interfaces → interfaces}/movie.interface.js +0 -0
- /package/{cjs/interfaces → interfaces}/search.interface.js +0 -0
- /package/{cjs/interfaces → interfaces}/user-ratings.interface.js +0 -0
- /package/{cjs/vars.js → vars.js} +0 -0
package/README.md
CHANGED
|
@@ -3,15 +3,16 @@
|
|
|
3
3
|
[](https://github.com/bartholomej/node-csfd-api/actions)
|
|
4
4
|
[](https://codecov.io/gh/bartholomej/node-csfd-api)
|
|
5
5
|
|
|
6
|
-
# CSFD API 🎥
|
|
6
|
+
# CSFD API 🎥 2025
|
|
7
7
|
|
|
8
8
|
> JavaScript NPM library for scraping **Czech Movie Database (csfd.cz)**
|
|
9
9
|
>
|
|
10
10
|
> - JavaScript / TypeScript
|
|
11
11
|
> - Browser + Node.js (SSR)
|
|
12
12
|
> - Tested (~100% Code coverage)
|
|
13
|
-
> - ✅ Ready for new ČSFD
|
|
13
|
+
> - ✅ Ready for new ČSFD 2025!
|
|
14
14
|
> - You can use in:
|
|
15
|
+
> - Docker – [_How to do it?_](#-docker)
|
|
15
16
|
> - Firebase function
|
|
16
17
|
> - AWS λ (lambda function)
|
|
17
18
|
> - CloudFlare Worker
|
|
@@ -361,6 +362,43 @@ csfd
|
|
|
361
362
|
|
|
362
363
|
_Note: You can not use both parameters `includesOnly` and `excludes`. Parameter `includesOnly` has a priority._
|
|
363
364
|
|
|
365
|
+
## 📦 Docker
|
|
366
|
+
|
|
367
|
+
You can use this library in Docker.
|
|
368
|
+
|
|
369
|
+
We have [prepared a Docker image](https://hub.docker.com/r/bartholomej/node-csfd-api) for you.
|
|
370
|
+
|
|
371
|
+
### Prebuilt image
|
|
372
|
+
|
|
373
|
+
```bash
|
|
374
|
+
docker pull bartholomej/node-csfd-api
|
|
375
|
+
```
|
|
376
|
+
|
|
377
|
+
### Build & run your own image
|
|
378
|
+
|
|
379
|
+
> Build image
|
|
380
|
+
|
|
381
|
+
```bash
|
|
382
|
+
docker build -t node-csfd-api .
|
|
383
|
+
```
|
|
384
|
+
|
|
385
|
+
> Run image on port 3000
|
|
386
|
+
|
|
387
|
+
```bash
|
|
388
|
+
docker run -p 3000:3000 node-csfd-api
|
|
389
|
+
```
|
|
390
|
+
|
|
391
|
+
> Open http://localhost:3000
|
|
392
|
+
|
|
393
|
+
### API endpoints
|
|
394
|
+
|
|
395
|
+
> Some examples
|
|
396
|
+
|
|
397
|
+
- `/movie/535121`
|
|
398
|
+
- `/search/quentin+tarantino`
|
|
399
|
+
- `/creator/2120`
|
|
400
|
+
- `/user-ratings/912-bart`
|
|
401
|
+
|
|
364
402
|
## 🧑💻 Used by
|
|
365
403
|
|
|
366
404
|
### Web extensions
|
|
@@ -463,7 +501,7 @@ That's why, with node-csfd-api, what happens on your device stays on your device
|
|
|
463
501
|
|
|
464
502
|
## 📝 License
|
|
465
503
|
|
|
466
|
-
Copyright © 2020 –
|
|
504
|
+
Copyright © 2020 – 2025 [Lukas Bartak](http://bartweb.cz)
|
|
467
505
|
|
|
468
506
|
Proudly powered by nature 🗻, wind 💨, tea 🍵 and beer 🍺 ;)
|
|
469
507
|
|
package/esm/fetchers/index.js
CHANGED
|
@@ -1,12 +1,3 @@
|
|
|
1
|
-
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
2
|
-
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
3
|
-
return new (P || (P = Promise))(function (resolve, reject) {
|
|
4
|
-
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
5
|
-
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
6
|
-
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
7
|
-
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
8
|
-
});
|
|
9
|
-
};
|
|
10
1
|
import { fetchSafe } from './fetch.polyfill';
|
|
11
2
|
const USER_AGENTS = [
|
|
12
3
|
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36',
|
|
@@ -17,16 +8,16 @@ const USER_AGENTS = [
|
|
|
17
8
|
const headers = {
|
|
18
9
|
'User-Agent': USER_AGENTS[Math.floor(Math.random() * USER_AGENTS.length)]
|
|
19
10
|
};
|
|
20
|
-
export const fetchPage = (url) =>
|
|
11
|
+
export const fetchPage = async (url) => {
|
|
21
12
|
try {
|
|
22
|
-
const response =
|
|
13
|
+
const response = await fetchSafe(url, { headers });
|
|
23
14
|
if (response.status >= 400 && response.status < 600) {
|
|
24
15
|
throw new Error(`node-csfd-api: Bad response ${response.status} for url: ${url}`);
|
|
25
16
|
}
|
|
26
|
-
return
|
|
17
|
+
return await response.text();
|
|
27
18
|
}
|
|
28
19
|
catch (e) {
|
|
29
20
|
console.error(e);
|
|
30
21
|
return 'Error';
|
|
31
22
|
}
|
|
32
|
-
}
|
|
23
|
+
};
|
|
@@ -13,18 +13,29 @@ export const getId = (url) => {
|
|
|
13
13
|
}
|
|
14
14
|
return null;
|
|
15
15
|
};
|
|
16
|
-
export const getName = (el) => {
|
|
17
|
-
return el.querySelector('h1').innerText.trim();
|
|
18
|
-
};
|
|
19
16
|
export const getCoords = (el) => {
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
const
|
|
23
|
-
|
|
17
|
+
if (!el)
|
|
18
|
+
return null;
|
|
19
|
+
const linkMapsEl = el.querySelector('a[href*="q="]');
|
|
20
|
+
if (!linkMapsEl)
|
|
21
|
+
return null;
|
|
22
|
+
const linkMaps = linkMapsEl.getAttribute('href');
|
|
23
|
+
const [_, latLng] = linkMaps.split('q=');
|
|
24
|
+
const coords = latLng.split(',');
|
|
25
|
+
if (coords.length !== 2)
|
|
26
|
+
return null;
|
|
27
|
+
const lat = Number(coords[0]);
|
|
28
|
+
const lng = Number(coords[1]);
|
|
29
|
+
if (Number.isFinite(lat) && Number.isFinite(lng)) {
|
|
30
|
+
return { lat, lng };
|
|
31
|
+
}
|
|
32
|
+
return null;
|
|
24
33
|
};
|
|
25
34
|
export const getCinemaUrl = (el) => {
|
|
26
|
-
var _a;
|
|
27
|
-
|
|
35
|
+
var _a, _b;
|
|
36
|
+
if (!el)
|
|
37
|
+
return '';
|
|
38
|
+
return (_b = (_a = el.querySelector('a[title="Přejít na webovou stránku kina"]')) === null || _a === void 0 ? void 0 : _a.attributes.href) !== null && _b !== void 0 ? _b : '';
|
|
28
39
|
};
|
|
29
40
|
export const parseCinema = (el) => {
|
|
30
41
|
const title = el.querySelector('.box-header h2').innerText.trim();
|
|
@@ -37,8 +48,9 @@ export const getGroupedFilmsByDate = (el) => {
|
|
|
37
48
|
.map((_, index) => index)
|
|
38
49
|
.filter((index) => index % 2 === 0)
|
|
39
50
|
.map((index) => {
|
|
51
|
+
var _a, _b, _c;
|
|
40
52
|
const [date, films] = divs.slice(index, index + 2);
|
|
41
|
-
const dateText = date === null || date === void 0 ? void 0 : date.
|
|
53
|
+
const dateText = (_c = (_b = (_a = date === null || date === void 0 ? void 0 : date.firstChild) === null || _a === void 0 ? void 0 : _a.textContent) === null || _b === void 0 ? void 0 : _b.trim()) !== null && _c !== void 0 ? _c : null;
|
|
42
54
|
return { date: dateText, films: getFilms('', films) };
|
|
43
55
|
});
|
|
44
56
|
return getDatesAndFilms;
|
|
@@ -12,23 +12,24 @@ export const getName = (el) => {
|
|
|
12
12
|
return el.querySelector('h1').innerText.trim();
|
|
13
13
|
};
|
|
14
14
|
export const getBirthdayInfo = (el) => {
|
|
15
|
-
|
|
16
|
-
const
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
15
|
+
var _a, _b;
|
|
16
|
+
const infoBlock = el.querySelector('h1 + p');
|
|
17
|
+
const text = infoBlock === null || infoBlock === void 0 ? void 0 : infoBlock.innerHTML.trim();
|
|
18
|
+
const birthPlaceRow = (_a = infoBlock === null || infoBlock === void 0 ? void 0 : infoBlock.querySelector('.info-place')) === null || _a === void 0 ? void 0 : _a.innerHTML.trim();
|
|
19
|
+
const ageRow = (_b = infoBlock === null || infoBlock === void 0 ? void 0 : infoBlock.querySelector('.info')) === null || _b === void 0 ? void 0 : _b.innerHTML.trim();
|
|
20
|
+
let birthday = '';
|
|
21
|
+
if (text) {
|
|
22
|
+
const parts = text.split('\n');
|
|
21
23
|
const birthdayRow = parts.find((x) => x.includes('nar.'));
|
|
22
|
-
const ageRow = parts.find((x) => x.includes('let)'));
|
|
23
|
-
const birthPlaceRow = parts.find((x) => x.includes('<br>')); // Ugly but there is no other way to detect
|
|
24
24
|
birthday = birthdayRow ? parseBirthday(birthdayRow) : '';
|
|
25
|
-
age = ageRow ? +parseAge(ageRow) : null;
|
|
26
|
-
birthPlace = birthPlaceRow ? parseBirthPlace(birthPlaceRow) : '';
|
|
27
25
|
}
|
|
26
|
+
const age = ageRow ? +parseAge(ageRow) : null;
|
|
27
|
+
const birthPlace = birthPlaceRow ? parseBirthPlace(birthPlaceRow) : '';
|
|
28
28
|
return { birthday, age, birthPlace };
|
|
29
29
|
};
|
|
30
30
|
export const getBio = (el) => {
|
|
31
|
-
|
|
31
|
+
var _a;
|
|
32
|
+
return ((_a = el.querySelector('.article-content p')) === null || _a === void 0 ? void 0 : _a.text.trim().split('\n')[0].trim()) || null;
|
|
32
33
|
};
|
|
33
34
|
export const getPhoto = (el) => {
|
|
34
35
|
const image = el.querySelector('img').attributes.src;
|
|
@@ -21,9 +21,11 @@ export const getPoster = (el) => {
|
|
|
21
21
|
return addProtocol(image);
|
|
22
22
|
};
|
|
23
23
|
export const getOrigins = (el) => {
|
|
24
|
-
var _a;
|
|
25
|
-
const originsRaw = el.querySelector('.article-content p .info').text;
|
|
26
|
-
|
|
24
|
+
var _a, _b;
|
|
25
|
+
const originsRaw = (_a = el.querySelector('.article-content p .info')) === null || _a === void 0 ? void 0 : _a.text;
|
|
26
|
+
if (!originsRaw)
|
|
27
|
+
return [];
|
|
28
|
+
const originsAll = (_b = originsRaw === null || originsRaw === void 0 ? void 0 : originsRaw.split(', ')) === null || _b === void 0 ? void 0 : _b[0];
|
|
27
29
|
return originsAll === null || originsAll === void 0 ? void 0 : originsAll.split('/').map((country) => country.trim());
|
|
28
30
|
};
|
|
29
31
|
export const parsePeople = (el, type) => {
|
|
@@ -16,7 +16,8 @@ export const getTitle = (el) => {
|
|
|
16
16
|
return el.querySelector('td.name .film-title-name').text;
|
|
17
17
|
};
|
|
18
18
|
export const getYear = (el) => {
|
|
19
|
-
|
|
19
|
+
var _a;
|
|
20
|
+
return +((_a = el.querySelectorAll('td.name .film-title-info .info')[0]) === null || _a === void 0 ? void 0 : _a.text.slice(1, -1)) || null;
|
|
20
21
|
};
|
|
21
22
|
export const getColorRating = (el) => {
|
|
22
23
|
const color = parseColor(el.querySelector('td.name .icon').classNames.split(' ').pop());
|
package/esm/index.js
CHANGED
|
@@ -1,12 +1,3 @@
|
|
|
1
|
-
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
2
|
-
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
3
|
-
return new (P || (P = Promise))(function (resolve, reject) {
|
|
4
|
-
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
5
|
-
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
6
|
-
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
7
|
-
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
8
|
-
});
|
|
9
|
-
};
|
|
10
1
|
import { CinemaScraper } from './services/cinema.service';
|
|
11
2
|
import { CreatorScraper } from './services/creator.service';
|
|
12
3
|
import { MovieScraper } from './services/movie.service';
|
|
@@ -20,30 +11,20 @@ export class Csfd {
|
|
|
20
11
|
this.searchService = searchService;
|
|
21
12
|
this.cinemaService = cinemaService;
|
|
22
13
|
}
|
|
23
|
-
userRatings(user, config) {
|
|
24
|
-
return
|
|
25
|
-
return this.userRatingsService.userRatings(user, config);
|
|
26
|
-
});
|
|
14
|
+
async userRatings(user, config) {
|
|
15
|
+
return this.userRatingsService.userRatings(user, config);
|
|
27
16
|
}
|
|
28
|
-
movie(movie) {
|
|
29
|
-
return
|
|
30
|
-
return this.movieService.movie(+movie);
|
|
31
|
-
});
|
|
17
|
+
async movie(movie) {
|
|
18
|
+
return this.movieService.movie(+movie);
|
|
32
19
|
}
|
|
33
|
-
creator(creator) {
|
|
34
|
-
return
|
|
35
|
-
return this.creatorService.creator(+creator);
|
|
36
|
-
});
|
|
20
|
+
async creator(creator) {
|
|
21
|
+
return this.creatorService.creator(+creator);
|
|
37
22
|
}
|
|
38
|
-
search(text) {
|
|
39
|
-
return
|
|
40
|
-
return this.searchService.search(text);
|
|
41
|
-
});
|
|
23
|
+
async search(text) {
|
|
24
|
+
return this.searchService.search(text);
|
|
42
25
|
}
|
|
43
|
-
cinema(district, period) {
|
|
44
|
-
return
|
|
45
|
-
return this.cinemaService.cinemas(+district, period);
|
|
46
|
-
});
|
|
26
|
+
async cinema(district, period) {
|
|
27
|
+
return this.cinemaService.cinemas(+district, period);
|
|
47
28
|
}
|
|
48
29
|
}
|
|
49
30
|
const movieScraper = new MovieScraper();
|
|
@@ -1,35 +1,24 @@
|
|
|
1
|
-
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
2
|
-
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
3
|
-
return new (P || (P = Promise))(function (resolve, reject) {
|
|
4
|
-
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
5
|
-
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
6
|
-
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
7
|
-
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
8
|
-
});
|
|
9
|
-
};
|
|
10
1
|
import { parse } from 'node-html-parser';
|
|
11
2
|
import { fetchPage } from '../fetchers';
|
|
12
3
|
import { cinemasUrl } from '../vars';
|
|
13
4
|
import { getCinemaId, getCinemaUrl, getCoords, getGroupedFilmsByDate, parseCinema } from './../helpers/cinema.helper';
|
|
14
5
|
export class CinemaScraper {
|
|
15
|
-
cinemas(district = 1, period = 'today') {
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
return this.cinema;
|
|
23
|
-
});
|
|
6
|
+
async cinemas(district = 1, period = 'today') {
|
|
7
|
+
const url = cinemasUrl(district, period);
|
|
8
|
+
const response = await fetchPage(url);
|
|
9
|
+
const cinemasHtml = parse(response);
|
|
10
|
+
const contentNode = cinemasHtml.querySelectorAll('#snippet--cinemas section.box');
|
|
11
|
+
this.buildCinemas(contentNode);
|
|
12
|
+
return this.cinema;
|
|
24
13
|
}
|
|
25
14
|
buildCinemas(contentNode) {
|
|
26
15
|
const cinemas = [];
|
|
27
|
-
contentNode.
|
|
28
|
-
|
|
16
|
+
contentNode.forEach((x) => {
|
|
17
|
+
const cinemaInfo = parseCinema(x);
|
|
29
18
|
const cinema = {
|
|
30
19
|
id: getCinemaId(x),
|
|
31
|
-
name:
|
|
32
|
-
city:
|
|
20
|
+
name: cinemaInfo === null || cinemaInfo === void 0 ? void 0 : cinemaInfo.name,
|
|
21
|
+
city: cinemaInfo === null || cinemaInfo === void 0 ? void 0 : cinemaInfo.city,
|
|
33
22
|
url: getCinemaUrl(x),
|
|
34
23
|
coords: getCoords(x),
|
|
35
24
|
screenings: getGroupedFilmsByDate(x)
|
|
@@ -1,27 +1,16 @@
|
|
|
1
|
-
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
2
|
-
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
3
|
-
return new (P || (P = Promise))(function (resolve, reject) {
|
|
4
|
-
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
5
|
-
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
6
|
-
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
7
|
-
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
8
|
-
});
|
|
9
|
-
};
|
|
10
1
|
import { parse } from 'node-html-parser';
|
|
11
2
|
import { fetchPage } from '../fetchers';
|
|
12
3
|
import { getBio, getBirthdayInfo, getFilms, getName, getPhoto } from '../helpers/creator.helper';
|
|
13
4
|
import { creatorUrl } from '../vars';
|
|
14
5
|
export class CreatorScraper {
|
|
15
|
-
creator(creatorId) {
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
return this.person;
|
|
24
|
-
});
|
|
6
|
+
async creator(creatorId) {
|
|
7
|
+
const url = creatorUrl(+creatorId);
|
|
8
|
+
const response = await fetchPage(url);
|
|
9
|
+
const creatorHtml = parse(response);
|
|
10
|
+
const asideNode = creatorHtml.querySelector('.creator-about');
|
|
11
|
+
const filmsNode = creatorHtml.querySelector('.creator-filmography');
|
|
12
|
+
this.buildCreator(+creatorId, asideNode, filmsNode);
|
|
13
|
+
return this.person;
|
|
25
14
|
}
|
|
26
15
|
buildCreator(id, asideEl, filmsNode) {
|
|
27
16
|
var _a, _b, _c;
|
|
@@ -1,29 +1,18 @@
|
|
|
1
|
-
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
2
|
-
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
3
|
-
return new (P || (P = Promise))(function (resolve, reject) {
|
|
4
|
-
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
5
|
-
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
6
|
-
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
7
|
-
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
8
|
-
});
|
|
9
|
-
};
|
|
10
1
|
import { parse } from 'node-html-parser';
|
|
11
2
|
import { fetchPage } from '../fetchers';
|
|
12
3
|
import { getBoxMovies, getColorRating, getDescriptions, getDuration, getGenres, getGroup, getOrigins, getPoster, getPremieres, getRandomPhoto, getRating, getRatingCount, getTags, getTitle, getTitlesOther, getTrivia, getType, getVods, getYear } from '../helpers/movie.helper';
|
|
13
4
|
import { movieUrl } from '../vars';
|
|
14
5
|
export class MovieScraper {
|
|
15
|
-
movie(movieId) {
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
return this.film;
|
|
26
|
-
});
|
|
6
|
+
async movie(movieId) {
|
|
7
|
+
const url = movieUrl(+movieId);
|
|
8
|
+
const response = await fetchPage(url);
|
|
9
|
+
const movieHtml = parse(response);
|
|
10
|
+
const pageClasses = movieHtml.querySelector('.page-content').classNames.split(' ');
|
|
11
|
+
const asideNode = movieHtml.querySelector('.aside-movie-profile');
|
|
12
|
+
const movieNode = movieHtml.querySelector('.main-movie-profile');
|
|
13
|
+
const jsonLd = movieHtml.querySelector('script[type="application/ld+json"]').innerText;
|
|
14
|
+
this.buildMovie(+movieId, movieNode, asideNode, pageClasses, jsonLd);
|
|
15
|
+
return this.film;
|
|
27
16
|
}
|
|
28
17
|
buildMovie(movieId, el, asideEl, pageClasses, jsonLd) {
|
|
29
18
|
this.film = {
|
|
@@ -1,12 +1,3 @@
|
|
|
1
|
-
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
2
|
-
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
3
|
-
return new (P || (P = Promise))(function (resolve, reject) {
|
|
4
|
-
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
5
|
-
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
6
|
-
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
7
|
-
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
8
|
-
});
|
|
9
|
-
};
|
|
10
1
|
import { parse } from 'node-html-parser';
|
|
11
2
|
import { fetchPage } from '../fetchers';
|
|
12
3
|
import { parseIdFromUrl } from '../helpers/global.helper';
|
|
@@ -14,22 +5,20 @@ import { getAvatar, getUser, getUserRealName, getUserUrl } from '../helpers/sear
|
|
|
14
5
|
import { getColorRating, getOrigins, getPoster, getTitle, getType, getUrl, getYear, parsePeople } from '../helpers/search.helper';
|
|
15
6
|
import { searchUrl } from '../vars';
|
|
16
7
|
export class SearchScraper {
|
|
17
|
-
search(text) {
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
return this.parseSearch(moviesNode, usersNode, tvSeriesNode);
|
|
26
|
-
});
|
|
8
|
+
async search(text) {
|
|
9
|
+
const url = searchUrl(text);
|
|
10
|
+
const response = await fetchPage(url);
|
|
11
|
+
const html = parse(response);
|
|
12
|
+
const moviesNode = html.querySelectorAll('.main-movies article');
|
|
13
|
+
const usersNode = html.querySelectorAll('.main-users article');
|
|
14
|
+
const tvSeriesNode = html.querySelectorAll('.main-series article');
|
|
15
|
+
return this.parseSearch(moviesNode, usersNode, tvSeriesNode);
|
|
27
16
|
}
|
|
28
17
|
parseSearch(moviesNode, usersNode, tvSeriesNode) {
|
|
29
18
|
const movies = [];
|
|
30
19
|
const users = [];
|
|
31
20
|
const tvSeries = [];
|
|
32
|
-
moviesNode.
|
|
21
|
+
moviesNode.forEach((m) => {
|
|
33
22
|
const url = getUrl(m);
|
|
34
23
|
const movie = {
|
|
35
24
|
id: parseIdFromUrl(url),
|
|
@@ -47,7 +36,7 @@ export class SearchScraper {
|
|
|
47
36
|
};
|
|
48
37
|
movies.push(movie);
|
|
49
38
|
});
|
|
50
|
-
usersNode.
|
|
39
|
+
usersNode.forEach((m) => {
|
|
51
40
|
const url = getUserUrl(m);
|
|
52
41
|
const user = {
|
|
53
42
|
id: parseIdFromUrl(url),
|
|
@@ -58,7 +47,7 @@ export class SearchScraper {
|
|
|
58
47
|
};
|
|
59
48
|
users.push(user);
|
|
60
49
|
});
|
|
61
|
-
tvSeriesNode.
|
|
50
|
+
tvSeriesNode.forEach((m) => {
|
|
62
51
|
const url = getUrl(m);
|
|
63
52
|
const user = {
|
|
64
53
|
id: parseIdFromUrl(url),
|
|
@@ -1,12 +1,3 @@
|
|
|
1
|
-
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
2
|
-
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
3
|
-
return new (P || (P = Promise))(function (resolve, reject) {
|
|
4
|
-
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
5
|
-
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
6
|
-
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
7
|
-
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
8
|
-
});
|
|
9
|
-
};
|
|
10
1
|
import { parse } from 'node-html-parser';
|
|
11
2
|
import { fetchPage } from '../fetchers';
|
|
12
3
|
import { getColorRating, getDate, getId, getTitle, getType, getUrl, getUserRating, getYear, sleep } from '../helpers/user-ratings.helper';
|
|
@@ -15,36 +6,34 @@ export class UserRatingsScraper {
|
|
|
15
6
|
constructor() {
|
|
16
7
|
this.films = [];
|
|
17
8
|
}
|
|
18
|
-
userRatings(user, config) {
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
yield sleep(config.allPagesDelay);
|
|
42
|
-
}
|
|
9
|
+
async userRatings(user, config) {
|
|
10
|
+
let allMovies = [];
|
|
11
|
+
const url = userRatingsUrl(user);
|
|
12
|
+
const response = await fetchPage(url);
|
|
13
|
+
const items = parse(response);
|
|
14
|
+
const movies = items.querySelectorAll('.box-user-rating .table-container tbody tr');
|
|
15
|
+
// Get number of pages
|
|
16
|
+
const pagesNode = items.querySelector('.pagination');
|
|
17
|
+
const pages = +(pagesNode === null || pagesNode === void 0 ? void 0 : pagesNode.childNodes[pagesNode.childNodes.length - 4].rawText) || 1;
|
|
18
|
+
allMovies = this.getPage(config, movies);
|
|
19
|
+
if (config === null || config === void 0 ? void 0 : config.allPages) {
|
|
20
|
+
console.log('User', user, url);
|
|
21
|
+
console.log('Fetching all pages', pages);
|
|
22
|
+
for (let i = 2; i <= pages; i++) {
|
|
23
|
+
console.log('Fetching page', i, 'out of', pages, '...');
|
|
24
|
+
const url = userRatingsUrl(user, i);
|
|
25
|
+
const response = await fetchPage(url);
|
|
26
|
+
const items = parse(response);
|
|
27
|
+
const movies = items.querySelectorAll('.box-user-rating .table-container tbody tr');
|
|
28
|
+
allMovies = [...this.getPage(config, movies)];
|
|
29
|
+
// Sleep
|
|
30
|
+
if (config.allPagesDelay) {
|
|
31
|
+
await sleep(config.allPagesDelay);
|
|
43
32
|
}
|
|
44
|
-
return allMovies;
|
|
45
33
|
}
|
|
46
34
|
return allMovies;
|
|
47
|
-
}
|
|
35
|
+
}
|
|
36
|
+
return allMovies;
|
|
48
37
|
}
|
|
49
38
|
getPage(config, movies) {
|
|
50
39
|
var _a, _b, _c, _d;
|
|
@@ -3,12 +3,11 @@ import { HTMLElement } from 'node-html-parser';
|
|
|
3
3
|
import { CSFDColorRating } from '../interfaces/global';
|
|
4
4
|
export declare const getColorRating: (el: HTMLElement) => CSFDColorRating;
|
|
5
5
|
export declare const getCinemaId: (el: HTMLElement | null) => number;
|
|
6
|
-
export declare const getId: (url: string) => number;
|
|
7
|
-
export declare const getName: (el: HTMLElement | null) => string;
|
|
6
|
+
export declare const getId: (url: string) => number | null;
|
|
8
7
|
export declare const getCoords: (el: HTMLElement | null) => {
|
|
9
8
|
lat: number;
|
|
10
9
|
lng: number;
|
|
11
|
-
};
|
|
10
|
+
} | null;
|
|
12
11
|
export declare const getCinemaUrl: (el: HTMLElement | null) => string;
|
|
13
12
|
export declare const parseCinema: (el: HTMLElement | null) => {
|
|
14
13
|
city: string;
|
|
@@ -8,4 +8,4 @@ export declare const getUrl: (el: HTMLElement) => string;
|
|
|
8
8
|
export declare const getColorRating: (el: HTMLElement) => CSFDColorRating;
|
|
9
9
|
export declare const getPoster: (el: HTMLElement) => string;
|
|
10
10
|
export declare const getOrigins: (el: HTMLElement) => string[];
|
|
11
|
-
export declare const parsePeople: (el: HTMLElement, type:
|
|
11
|
+
export declare const parsePeople: (el: HTMLElement, type: "directors" | "actors") => CSFDCreator[];
|
|
@@ -19,6 +19,6 @@ export declare class Csfd {
|
|
|
19
19
|
movie(movie: number): Promise<CSFDMovie>;
|
|
20
20
|
creator(creator: number): Promise<CSFDCreator>;
|
|
21
21
|
search(text: string): Promise<CSFDSearch>;
|
|
22
|
-
cinema(district: number, period: CSFDCinemaPeriod): Promise<CSFDCinema[]>;
|
|
22
|
+
cinema(district: number | string, period: CSFDCinemaPeriod): Promise<CSFDCinema[]>;
|
|
23
23
|
}
|
|
24
24
|
export declare const csfd: Csfd;
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export declare const fetchSafe: typeof fetch;
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export declare const fetchPage: (url: string) => Promise<string>;
|
|
@@ -1,13 +1,4 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
-
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
3
|
-
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
4
|
-
return new (P || (P = Promise))(function (resolve, reject) {
|
|
5
|
-
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
6
|
-
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
7
|
-
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
8
|
-
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
9
|
-
});
|
|
10
|
-
};
|
|
11
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
12
3
|
exports.fetchPage = void 0;
|
|
13
4
|
const fetch_polyfill_1 = require("./fetch.polyfill");
|
|
@@ -20,17 +11,17 @@ const USER_AGENTS = [
|
|
|
20
11
|
const headers = {
|
|
21
12
|
'User-Agent': USER_AGENTS[Math.floor(Math.random() * USER_AGENTS.length)]
|
|
22
13
|
};
|
|
23
|
-
const fetchPage = (url) =>
|
|
14
|
+
const fetchPage = async (url) => {
|
|
24
15
|
try {
|
|
25
|
-
const response =
|
|
16
|
+
const response = await (0, fetch_polyfill_1.fetchSafe)(url, { headers });
|
|
26
17
|
if (response.status >= 400 && response.status < 600) {
|
|
27
18
|
throw new Error(`node-csfd-api: Bad response ${response.status} for url: ${url}`);
|
|
28
19
|
}
|
|
29
|
-
return
|
|
20
|
+
return await response.text();
|
|
30
21
|
}
|
|
31
22
|
catch (e) {
|
|
32
23
|
console.error(e);
|
|
33
24
|
return 'Error';
|
|
34
25
|
}
|
|
35
|
-
}
|
|
26
|
+
};
|
|
36
27
|
exports.fetchPage = fetchPage;
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import { CSFDCinemaGroupedFilmsByDate, CSFDCinemaMeta, CSFDCinemaMovie } from 'interfaces/cinema.interface';
|
|
2
|
+
import { HTMLElement } from 'node-html-parser';
|
|
3
|
+
import { CSFDColorRating } from '../interfaces/global';
|
|
4
|
+
export declare const getColorRating: (el: HTMLElement) => CSFDColorRating;
|
|
5
|
+
export declare const getCinemaId: (el: HTMLElement | null) => number;
|
|
6
|
+
export declare const getId: (url: string) => number | null;
|
|
7
|
+
export declare const getCoords: (el: HTMLElement | null) => {
|
|
8
|
+
lat: number;
|
|
9
|
+
lng: number;
|
|
10
|
+
} | null;
|
|
11
|
+
export declare const getCinemaUrl: (el: HTMLElement | null) => string;
|
|
12
|
+
export declare const parseCinema: (el: HTMLElement | null) => {
|
|
13
|
+
city: string;
|
|
14
|
+
name: string;
|
|
15
|
+
};
|
|
16
|
+
export declare const getGroupedFilmsByDate: (el: HTMLElement | null) => CSFDCinemaGroupedFilmsByDate[];
|
|
17
|
+
export declare const getFilms: (date: string, el: HTMLElement | null) => CSFDCinemaMovie[];
|
|
18
|
+
export declare const parseMeta: (meta: string[]) => CSFDCinemaMeta[];
|