node-csfd-api 1.11.0-next.0 → 2.1.0-next.1

package/README.md

@@ -5,33 +5,33 @@
 
 # CSFD API 2021
 
-> Tiny NPM library for scraping csfd.cz
+> JavaScript NPM library for scraping **Czech Movie Database (csfd.cz)**
 >
 > - Browser + Node.js (SSR)
-> - TypeScript 100%
-> - ✅ Ready for CSFD 2021!
-
-Scraper:
-
-> - [Movies and TV Series](#Movie)
-> - [User Ratings](#User-Ratings)
-> - [Search](#Search)
+> - JavaScript / TypeScript
+> - Tested (~100% Code coverage)
+> - ✅ Ready for new ČSFD 2021!
+> - You can use in:
+>   - Firebase function
+>   - AWS λ (lambda function)
+>   - Chrome extension
+>   - React native app
+>   - ...
 
 ## Install
 
-via yarn
-
 ```bash
-yarn add node-csfd-api
+npm install node-csfd-api --save
+# yarn add node-csfd-api
 ```
 
-via npm
+## Usage and examples
 
-```bash
-npm install node-csfd-api
-```
+- [Movies and TV Series](#Movie)
+- [User Ratings](#User-Ratings)
+- [Search](#Search)
 
-## Movie
+### Movie
 
 Get info about [this movie](https://www.csfd.cz/film/535121-na-spatne-strane/komentare/) _(id: 535121)_
 
@@ -41,7 +41,7 @@ import { csfd } from 'node-csfd-api';
 csfd.movie(535121).then((movie) => console.log(movie));
 ```
 
-### Results
+#### Results
 
 ```javascript
 {
@@ -66,7 +66,7 @@ csfd.movie(535121).then((movie) => console.log(movie));
     { country: 'Austrálie', title: 'Dragged Across Concrete' },
     { country: 'Velká Británie', title: 'Dragged Across Concrete' }
   ],
-  poster: '//image.pmgstatic.com/cache/resized/w1080/files/images/film/posters/163/579/163579352_bf8737.jpg',
+  poster: 'https://image.pmgstatic.com/cache/resized/w1080/files/images/film/posters/163/579/163579352_bf8737.jpg',
   creators: {
     directors: [
     {
@@ -133,7 +133,7 @@ csfd.movie(535121).then((movie) => console.log(movie));
 }
 ```
 
-## Search
+### Search
 
 > Search movies and users
 
@@ -143,7 +143,7 @@ import { csfd } from 'node-csfd-api';
 csfd.search('bart').then((search) => console.log(search));
 ```
 
-### Results
+#### Results
 
 ```javascript
 movies: [
@@ -175,13 +175,13 @@ users: [
     id: 912,
     user: 'BART!',
     userRealName: 'Lukáš Barták',
-    avatar: '//image.pmgstatic.com/cache/resized/w45h60/files/images/user/avatars/000/281/281554_1c0fef.jpg',
+    avatar: 'https://image.pmgstatic.com/cache/resized/w45h60/files/images/user/avatars/000/281/281554_1c0fef.jpg',
     url: 'https://www.csfd.cz/uzivatel/912-bart/'
   }
 ]
 ```
 
-## Creators
+### Creators
 
 > Search creators and filmography
 
@@ -191,7 +191,7 @@ import { csfd } from 'node-csfd-api';
 csfd.creator(2120).then((creator) => console.log(creator));
 ```
 
-### Results
+#### Results
 
 ```javascript
 {
@@ -199,7 +199,7 @@ csfd.creator(2120).then((creator) => console.log(creator));
   name: 'Quentin Tarantino',
   birthday: '27.03.1963',
   birthplace: 'Knoxville, Tennessee, USA',
-  photo: '//image.pmgstatic.com/cache/resized/w100h132crop/files/images/creator/photos/164/515/164515525_b98f8a.jpg',
+  photo: 'https://image.pmgstatic.com/cache/resized/w100h132crop/files/images/creator/photos/164/515/164515525_b98f8a.jpg',
   age: 58,
   bio: 'Quentin Tarantino se narodil 27. března roku 1963 v americkém Knoxville teprve šestnáctileté Connie Tarantinové. Své jméno Quentin dostal podle matčiny oblíbené televizní postavy Quinta ze seriálu "Gunsmoke". Quentinův otec byl jistý Tony Tarantino, který rodinu opustil když byl Quentin ještě malinký. Jeho dětství a dospívání ovlivnily nejen filmy, ale pop kultura obecně. Televizní seriály, komiksy, populární hudba, to vše jako mladý hltal ve velkém a stále neměl…',
   films: [
@@ -268,9 +268,9 @@ csfd.creator(2120).then((creator) => console.log(creator));
 }
 ```
 
-## User Ratings
+### User Ratings
 
-### Last ratings (last page)
+#### Last ratings (last page)
 
 Get [my last 50 ratings](https://www.csfd.cz/uzivatel/912-bart/hodnoceni/)
 
@@ -280,7 +280,7 @@ import { csfd } from 'node-csfd-api';
 csfd.userRatings('912-bart').then((ratings) => console.log(ratings));
 ```
 
-### All ratings (all pages)
+#### All ratings (all pages)
 
 > Warning: Use it wisely. Can be detected and banned. Consider using it together with `allPagesDelay` attribute.
 
@@ -297,7 +297,7 @@ csfd
   .then((ratings) => console.log(ratings));
 ```
 
-### Results
+#### Results
 
 ```javascript
 [
@@ -322,14 +322,14 @@ csfd
 ];
 ```
 
-### Options
+#### Options
 
-| Option            | Type                                                                                                                                       | Default | Description                                               |
-| ----------------- | ------------------------------------------------------------------------------------------------------------------------------------------ | ------- | --------------------------------------------------------- |
-| **includesOnly**  | [CSFDFilmTypes[]](https://github.com/bartholomej/node-csfd-api/blob/8fa5f9cbc7e7f2b62b0bd2c2b5a24c9a63444f6a/src/interfaces/global.ts#L25) | null    | Including only film types. eg. `['TV Seriál', 'koncert']` |
-| **exclude**       | [CSFDFilmTypes[]](https://github.com/bartholomej/node-csfd-api/blob/8fa5f9cbc7e7f2b62b0bd2c2b5a24c9a63444f6a/src/interfaces/global.ts#L25) | null    | Excluding film types eg. `['epizoda', 'série']`           |
-| **allPages**      | boolean                                                                                                                                    | false   | Get all pages                                             |
-| **allPagesDelay** | number                                                                                                                                     | 0       | Delay on each page request. In milliseconds               |
+| Option            | Type                                                                                                                                       | Default | Description                                            |
+| ----------------- | ------------------------------------------------------------------------------------------------------------------------------------------ | ------- | ------------------------------------------------------ |
+| **includesOnly**  | [CSFDFilmTypes[]](https://github.com/bartholomej/node-csfd-api/blob/8fa5f9cbc7e7f2b62b0bd2c2b5a24c9a63444f6a/src/interfaces/global.ts#L25) | null    | Including only film types. eg. `['seriál', 'koncert']` |
+| **exclude**       | [CSFDFilmTypes[]](https://github.com/bartholomej/node-csfd-api/blob/8fa5f9cbc7e7f2b62b0bd2c2b5a24c9a63444f6a/src/interfaces/global.ts#L25) | null    | Excluding film types eg. `['epizoda', 'série']`        |
+| **allPages**      | boolean                                                                                                                                    | false   | Get all pages                                          |
+| **allPagesDelay** | number                                                                                                                                     | 0       | Delay on each page request. In milliseconds            |
 
 _Note: You can not use both parameters 'includesOnly' and 'excludes'. Parameter 'includesOnly' has a priority._
 

package/fetchers/index.js

@@ -4,7 +4,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.fetchPage = void 0;
-const node_fetch_1 = __importDefault(require("node-fetch"));
+const cross_fetch_1 = __importDefault(require("cross-fetch"));
 const USER_AGENTS = [
     'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36',
     'Mozilla/5.0 (iPhone; CPU iPhone OS 14_3 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) CriOS/87.0.4280.77 Mobile/15E148 Safari/604.1',
@@ -16,7 +16,7 @@ const headers = {
 };
 const fetchPage = async (url) => {
     try {
-        const response = await (0, node_fetch_1.default)(url, { headers });
+        const response = await (0, cross_fetch_1.default)(url, { headers });
         if (response.status >= 400 && response.status < 600) {
             throw new Error(`node-csfd-api: Bad response ${response.status} for url: ${url}`);
         }

package/helpers/creator.helper.js

@@ -39,7 +39,8 @@ const getBio = (el) => {
 };
 exports.getBio = getBio;
 const getPhoto = (el) => {
-    return el.querySelector('img').attributes.src;
+    const image = el.querySelector('img').attributes.src;
+    return (0, global_helper_1.addProtocol)(image);
 };
 exports.getPhoto = getPhoto;
 const parseBirthday = (text) => {

package/helpers/global.helper.d.ts

@@ -3,3 +3,5 @@ import { Colors } from '../interfaces/user-ratings.interface';
 export declare const parseIdFromUrl: (url: string) => number;
 export declare const getColor: (cls: string) => CSFDColorRating;
 export declare const parseColor: (quality: Colors) => CSFDColorRating;
+export declare const addProtocol: (url: string) => string;
+export declare const parseISO8601Duration: (iso: string) => number;

package/helpers/global.helper.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.parseColor = exports.getColor = exports.parseIdFromUrl = void 0;
+exports.parseISO8601Duration = exports.addProtocol = exports.parseColor = exports.getColor = exports.parseIdFromUrl = void 0;
 const parseIdFromUrl = (url) => {
     const idSlug = url.split('/')[2];
     const id = idSlug.split('-')[0];
@@ -37,3 +37,23 @@ const parseColor = (quality) => {
     }
 };
 exports.parseColor = parseColor;
+const addProtocol = (url) => {
+    return url.startsWith('//') ? 'https:' + url : url;
+};
+exports.addProtocol = addProtocol;
+const parseISO8601Duration = (iso) => {
+    const iso8601DurationRegex = /(-)?P(?:([.,\d]+)Y)?(?:([.,\d]+)M)?(?:([.,\d]+)W)?(?:([.,\d]+)D)?T(?:([.,\d]+)H)?(?:([.,\d]+)M)?(?:([.,\d]+)S)?/;
+    const matches = iso.match(iso8601DurationRegex);
+    const duration = {
+        sign: matches[1] === undefined ? '+' : '-',
+        years: matches[2] === undefined ? 0 : matches[2],
+        months: matches[3] === undefined ? 0 : matches[3],
+        weeks: matches[4] === undefined ? 0 : matches[4],
+        days: matches[5] === undefined ? 0 : matches[5],
+        hours: matches[6] === undefined ? 0 : matches[6],
+        minutes: matches[7] === undefined ? 0 : matches[7],
+        seconds: matches[8] === undefined ? 0 : matches[8]
+    };
+    return +duration.minutes;
+};
+exports.parseISO8601Duration = parseISO8601Duration;

package/helpers/movie.helper.d.ts

@@ -8,12 +8,11 @@ export declare const getOrigins: (el: HTMLElement) => string[];
 export declare const getColorRating: (bodyClasses: string[]) => CSFDColorRating;
 export declare const getRating: (el: HTMLElement) => number;
 export declare const getRatingCount: (el: HTMLElement) => number;
-export declare const getYear: (el: HTMLElement) => string | number;
-export declare const getDuration: (el: HTMLElement) => number;
+export declare const getYear: (el: string) => number;
+export declare const getDuration: (jsonLdRaw: string, el: HTMLElement) => number;
 export declare const getTitlesOther: (el: HTMLElement) => CSFDTitlesOther[];
 export declare const getPoster: (el: HTMLElement) => string;
 export declare const getDescriptions: (el: HTMLElement) => string[];
-export declare const getDirectors: (el: HTMLElement) => CSFDCreator[];
 export declare const parsePeople: (el: HTMLElement) => CSFDCreator[];
 export declare const getGroup: (el: HTMLElement, group: CSFDCreatorGroups) => CSFDCreator[];
 export declare const getType: (el: HTMLElement) => string;

package/helpers/movie.helper.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.getTags = exports.getPremieres = exports.getBoxMovies = exports.getBoxContent = exports.getVods = exports.getType = exports.getGroup = exports.parsePeople = exports.getDirectors = exports.getDescriptions = exports.getPoster = exports.getTitlesOther = exports.getDuration = exports.getYear = exports.getRatingCount = exports.getRating = exports.getColorRating = exports.getOrigins = exports.getGenres = exports.getTitle = exports.getId = void 0;
+exports.getTags = exports.getPremieres = exports.getBoxMovies = exports.getBoxContent = exports.getVods = exports.getType = exports.getGroup = exports.parsePeople = exports.getDescriptions = exports.getPoster = exports.getTitlesOther = exports.getDuration = exports.getYear = exports.getRatingCount = exports.getRating = exports.getColorRating = exports.getOrigins = exports.getGenres = exports.getTitle = exports.getId = void 0;
 const global_helper_1 = require("./global.helper");
 const getId = (el) => {
     const url = el.querySelector('.tabs .tab-nav-list a').attributes.href;
@@ -50,23 +50,41 @@ const getRatingCount = (el) => {
 };
 exports.getRatingCount = getRatingCount;
 const getYear = (el) => {
-    return el.querySelector('.origin span').innerText.replace(/[{()}]/g, '');
+    try {
+        const jsonLd = JSON.parse(el);
+        return +jsonLd.dateCreated;
+    }
+    catch (error) {
+        console.error('node-csfd-api: Error parsing JSON-LD', error);
+        return null;
+    }
 };
 exports.getYear = getYear;
-const getDuration = (el) => {
-    const origin = el.querySelector('.origin').innerText;
-    const timeString = origin.split(',');
-    if (timeString.length > 2) {
-        const timeString2 = timeString.pop().trim();
-        const timeRaw = timeString2.split('(')[0].trim();
-        const hoursMinsRaw = timeRaw.split('min')[0];
-        const hoursMins = hoursMinsRaw.split('h');
-        // Resolve hours + minutes format
-        const duration = hoursMins.length > 1 ? +hoursMins[0] * 60 + +hoursMins[1] : +hoursMins[0];
-        return duration;
+const getDuration = (jsonLdRaw, el) => {
+    let duration = null;
+    try {
+        const jsonLd = JSON.parse(jsonLdRaw);
+        duration = jsonLd.duration;
+        return (0, global_helper_1.parseISO8601Duration)(duration);
     }
-    else {
-        return null;
+    catch (error) {
+        const origin = el.querySelector('.origin').innerText;
+        const timeString = origin.split(',');
+        if (timeString.length > 2) {
+            // Get last time elelment
+            const timeString2 = timeString.pop().trim();
+            // Clean it
+            const timeRaw = timeString2.split('(')[0].trim();
+            // Split by minutes and hours
+            const hoursMinsRaw = timeRaw.split('min')[0];
+            const hoursMins = hoursMinsRaw.split('h');
+            // Resolve hours + minutes format
+            duration = hoursMins.length > 1 ? +hoursMins[0] * 60 + +hoursMins[1] : +hoursMins[0];
+            return duration;
+        }
+        else {
+            return null;
+        }
     }
 };
 exports.getDuration = getDuration;
@@ -98,7 +116,8 @@ const getPoster = (el) => {
         else {
             // Full sized image (not thumb)
             const imageThumb = poster.attributes.src.split('?')[0];
-            return imageThumb.replace(/\/w140\//, '/w1080/');
+            const image = imageThumb.replace(/\/w140\//, '/w1080/');
+            return (0, global_helper_1.addProtocol)(image);
         }
     }
     else {
@@ -114,12 +133,6 @@ const getDescriptions = (el) => {
     return plot ? [plot] : [];
 };
 exports.getDescriptions = getDescriptions;
-const getDirectors = (el) => {
-    const creators = el.querySelectorAll('.creators span');
-    const directorsSpan = creators.filter((creator) => creator.attributes.itemprop === 'director')[0];
-    return (directorsSpan && (0, exports.parsePeople)(directorsSpan)) || [];
-};
-exports.getDirectors = getDirectors;
 const parsePeople = (el) => {
     const people = el.querySelectorAll('a');
     return (people

package/helpers/search-user.helper.js

@@ -1,6 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.getUserUrl = exports.getAvatar = exports.getUserRealName = exports.getUser = void 0;
+const global_helper_1 = require("./global.helper");
 const getUser = (el) => {
     return el.querySelector('.user-title-name').text;
 };
@@ -11,7 +12,8 @@ const getUserRealName = (el) => {
 };
 exports.getUserRealName = getUserRealName;
 const getAvatar = (el) => {
-    return el.querySelector('.article-img img').attributes.src;
+    const image = el.querySelector('.article-img img').attributes.src;
+    return (0, global_helper_1.addProtocol)(image);
 };
 exports.getAvatar = getAvatar;
 const getUserUrl = (el) => {

package/helpers/search.helper.d.ts

@@ -3,7 +3,7 @@ import { CSFDColorRating, CSFDFilmTypes } from '../interfaces/global';
 import { CSFDCreator } from '../interfaces/movie.interface';
 export declare const getType: (el: HTMLElement) => CSFDFilmTypes;
 export declare const getTitle: (el: HTMLElement) => string;
-export declare const getYear: (el: HTMLElement) => string;
+export declare const getYear: (el: HTMLElement) => number;
 export declare const getUrl: (el: HTMLElement) => string;
 export declare const getColorRating: (el: HTMLElement) => CSFDColorRating;
 export declare const getPoster: (el: HTMLElement) => string;

package/helpers/search.helper.js

@@ -13,7 +13,7 @@ const getTitle = (el) => {
 exports.getTitle = getTitle;
 const getYear = (el) => {
     var _a;
-    return (_a = el.querySelectorAll('.film-title-info .info')[0]) === null || _a === void 0 ? void 0 : _a.innerText.replace(/[{()}]/g, '');
+    return +((_a = el.querySelectorAll('.film-title-info .info')[0]) === null || _a === void 0 ? void 0 : _a.innerText.replace(/[{()}]/g, ''));
 };
 exports.getYear = getYear;
 const getUrl = (el) => {
@@ -25,14 +25,15 @@ const getColorRating = (el) => {
 };
 exports.getColorRating = getColorRating;
 const getPoster = (el) => {
-    return el.querySelector('img').attributes.src;
+    const image = el.querySelector('img').attributes.src;
+    return (0, global_helper_1.addProtocol)(image);
 };
 exports.getPoster = getPoster;
 const getOrigins = (el) => {
+    var _a;
     const originsRaw = el.querySelector('.article-content p .info').text;
-    const origins = originsRaw === null || originsRaw === void 0 ? void 0 : originsRaw.split(', ');
-    origins.pop();
-    return origins;
+    const originsAll = (_a = originsRaw === null || originsRaw === void 0 ? void 0 : originsRaw.split(', ')) === null || _a === void 0 ? void 0 : _a[0];
+    return originsAll === null || originsAll === void 0 ? void 0 : originsAll.split('/').map((country) => country.trim());
 };
 exports.getOrigins = getOrigins;
 const parsePeople = (el, type) => {

package/interfaces/global.d.ts

@@ -1,7 +1,7 @@
 export interface CSFDScreening {
     id: number;
     title: string;
-    year: number | string;
+    year: number;
     url: string;
     type: CSFDFilmTypes;
     /**
@@ -19,4 +19,4 @@ export interface CSFDScreening {
 }
 export declare type CSFDColorRating = 'bad' | 'average' | 'good' | 'unknown';
 export declare type CSFDStars = 0 | 1 | 2 | 3 | 4 | 5;
-export declare type CSFDFilmTypes = 'film' | 'TV film' | 'TV pořad' | 'TV seriál' | 'divadelní záznam' | 'koncert' | 'série' | 'studentský film' | 'amatérský film' | 'hudební videoklip' | 'epizoda';
+export declare type CSFDFilmTypes = 'film' | 'TV film' | 'pořad' | 'seriál' | 'divadelní záznam' | 'koncert' | 'série' | 'studentský film' | 'amatérský film' | 'hudební videoklip' | 'epizoda';

package/interfaces/movie.interface.d.ts

@@ -16,7 +16,7 @@ export interface CSFDMovie extends CSFDScreening {
     similar: CSFDMovieListItem[];
 }
 export interface CSFDVod {
-    title: 'Netflix' | 'hbogo' | 'Prime Video' | 'Apple TV+' | 'ivysilani' | 'Aerovod' | 'o2tv' | 'Voyo' | 'FILMY ČESKY A ZADARMO' | 'VAPET' | 'VOREL FILM' | string;
+    title: 'Netflix' | 'hbogo' | 'Prime Video' | 'Apple TV+' | 'iTunes' | 'Aerovod' | 'Edisonline' | 'o2tv' | 'SledovaniTV' | 'Starmax' | 'DAFilms' | 'FILMY ČESKY A ZADARMO' | 'Youtube Česká filmová klasika' | 'VAPET' | 'VOREL FILM' | 'ivysilani' | 'Google Play' | 'Voyo' | string;
     url: string;
 }
 export interface CSFDCreators {

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "node-csfd-api",
-  "version": "1.11.0-next.0",
-  "description": "Simple NPM library for scraping CSFD",
+  "version": "2.1.0-next.1",
+  "description": "ČSFD API in JavaScript. Amazing NPM library for scrapping csfd.cz :)",
   "main": "./index.js",
   "author": "BART! <bart@bartweb.cz>",
   "scripts": {
@@ -21,8 +21,8 @@
     "release:major": "git checkout master && npm version major -m \"chore(update): major release %s 💥\""
   },
   "dependencies": {
-    "node-fetch": "cjs",
-    "node-html-parser": "^5.1.0"
+    "cross-fetch": "^3.1.5",
+    "node-html-parser": "^5.2.0"
   },
   "repository": {
     "url": "git+https://github.com/bartholomej/node-csfd-api.git",
@@ -46,7 +46,7 @@
     "api"
   ],
   "engines": {
-    "node": ">= 10"
+    "node": ">= 12"
   },
   "license": "MIT",
   "module": "./index.js",

package/services/movie.service.js

@@ -13,15 +13,16 @@ class MovieScraper {
         const pageClasses = movieHtml.querySelector('.page-content').classNames.split(' ');
         const asideNode = movieHtml.querySelector('.aside-movie-profile');
         const movieNode = movieHtml.querySelector('.main-movie-profile');
-        this.buildMovie(+movieId, movieNode, asideNode, pageClasses);
+        const jsonLd = movieHtml.querySelector('script[type="application/ld+json"]').innerText;
+        this.buildMovie(+movieId, movieNode, asideNode, pageClasses, jsonLd);
         return this.film;
     }
-    buildMovie(movieId, el, asideEl, pageClasses) {
+    buildMovie(movieId, el, asideEl, pageClasses, jsonLd) {
         this.film = {
             id: movieId,
             title: (0, movie_helper_1.getTitle)(el),
-            year: (0, movie_helper_1.getYear)(el),
-            duration: (0, movie_helper_1.getDuration)(el),
+            year: (0, movie_helper_1.getYear)(jsonLd),
+            duration: (0, movie_helper_1.getDuration)(jsonLd, el),
             descriptions: (0, movie_helper_1.getDescriptions)(el),
             genres: (0, movie_helper_1.getGenres)(el),
             type: (0, movie_helper_1.getType)(el),
@@ -33,7 +34,7 @@ class MovieScraper {
             titlesOther: (0, movie_helper_1.getTitlesOther)(el),
             poster: (0, movie_helper_1.getPoster)(el),
             creators: {
-                directors: (0, movie_helper_1.getDirectors)(el),
+                directors: (0, movie_helper_1.getGroup)(el, 'Režie'),
                 writers: (0, movie_helper_1.getGroup)(el, 'Scénář'),
                 cinematography: (0, movie_helper_1.getGroup)(el, 'Kamera'),
                 music: (0, movie_helper_1.getGroup)(el, 'Hudba'),

package/test.js

@@ -1 +0,0 @@
-"use strict";