node-csfd-api-racintom 1.0.0

package/src/index.ts

@@ -0,0 +1,42 @@
+import { CSFDCreator } from './interfaces/creator.interface';
+import { CSFDMovie } from './interfaces/movie.interface';
+import { CSFDSearch } from './interfaces/search.interface';
+import { CSFDUserRatingConfig, CSFDUserRatings } from './interfaces/user-ratings.interface';
+import { CreatorScraper } from './services/creator.service';
+import { MovieScraper } from './services/movie.service';
+import { SearchScraper } from './services/search.service';
+import { UserRatingsScraper } from './services/user-ratings.service';
+
+export class Csfd {
+  constructor(
+    private userRatingsService: UserRatingsScraper,
+    private movieService: MovieScraper,
+    private creatorService: CreatorScraper,
+    private searchService: SearchScraper
+  ) {}
+
+  public async userRatings(
+    user: string | number,
+    config?: CSFDUserRatingConfig
+  ): Promise<CSFDUserRatings[]> {
+    return this.userRatingsService.userRatings(user, config);
+  }
+
+  public async movie(movie: number): Promise<CSFDMovie> {
+    return this.movieService.movie(+movie);
+  }
+
+  public async creator(creator: number): Promise<CSFDCreator> {
+    return this.creatorService.creator(+creator);
+  }
+
+  public async search(text: string): Promise<CSFDSearch> {
+    return this.searchService.search(text);
+  }
+}
+
+const movieScraper = new MovieScraper();
+const userRatingsScraper = new UserRatingsScraper();
+const creatorScraper = new CreatorScraper();
+const searchScraper = new SearchScraper();
+export const csfd = new Csfd(userRatingsScraper, movieScraper, creatorScraper, searchScraper);

package/src/interfaces/creator.interface.ts

@@ -0,0 +1,14 @@
+import { CSFDScreening } from './global';
+
+export interface CSFDCreator {
+  id: number;
+  name: string;
+  birthday: string;
+  birthplace: string;
+  photo: string;
+  age: number | string;
+  bio: string;
+  films: CSFDCreatorScreening[];
+}
+
+export type CSFDCreatorScreening = Omit<CSFDScreening, 'url' | 'type'>;

package/src/interfaces/global.ts

@@ -0,0 +1,36 @@
+export interface CSFDScreening {
+  id: number;
+  title: string;
+  year: number;
+  url: string;
+  type: CSFDFilmTypes;
+  /**
+   * Overall aggregated rating. (On the web usually represented by colors).
+   *
+   * 'unknown': unknown (gray color)
+   *
+   * 'good': 70% – 100 % (red color)
+   *
+   * 'average': 30% - 69% (blue color)
+   *
+   * 'bad': 0% - 29% (black color)
+   */
+  colorRating: CSFDColorRating;
+}
+
+export type CSFDColorRating = 'bad' | 'average' | 'good' | 'unknown';
+
+export type CSFDStars = 0 | 1 | 2 | 3 | 4 | 5;
+
+export type CSFDFilmTypes =
+  | 'film'
+  | 'TV film'
+  | 'pořad'
+  | 'seriál'
+  | 'divadelní záznam'
+  | 'koncert'
+  | 'série'
+  | 'studentský film'
+  | 'amatérský film'
+  | 'hudební videoklip'
+  | 'epizoda';

package/src/interfaces/movie.interface.ts

@@ -0,0 +1,157 @@
+import { CSFDScreening } from './global';
+
+export interface CSFDMovie extends CSFDScreening {
+  rating: number | null;
+  poster: string;
+  photo: string;
+  ratingCount: number | null;
+  duration: number | string;
+  titlesOther: CSFDTitlesOther[];
+  origins: string[];
+  descriptions: string[];
+  trivia: string[];
+  genres: CSFDGenres[] | string[];
+  creators: CSFDCreators;
+  vod: CSFDVod[];
+  tags: string[];
+  premieres: CSFDPremiere[];
+  related: CSFDMovieListItem[];
+  similar: CSFDMovieListItem[];
+  seasons: CSFDSeasons | null;
+}
+
+export type CSFDVodService =
+  | 'Netflix'
+  | 'hbogo'
+  | 'Prime Video'
+  | 'Apple TV+'
+  | 'iTunes'
+  | 'KVIFF.TV'
+  | 'Edisonline'
+  | 'o2tv'
+  | 'SledovaniTV'
+  | 'Starmax'
+  | 'DAFilms'
+  | 'FILMY ČESKY A ZADARMO'
+  | 'Youtube Česká filmová klasika'
+  | 'VAPET'
+  | 'VOREL FILM'
+  | 'ivysilani'
+  | 'Google Play'
+  | 'Voyo'
+  | 'DVD';
+
+export interface CSFDVod {
+  title: CSFDVodService;
+  url: string;
+}
+
+export interface CSFDCreators {
+  directors: CSFDCreator[];
+  writers: CSFDCreator[];
+  cinematography: CSFDCreator[];
+  music: CSFDCreator[];
+  actors: CSFDCreator[];
+  basedOn: CSFDCreator[];
+  producers: CSFDCreator[];
+  filmEditing: CSFDCreator[];
+  costumeDesign: CSFDCreator[];
+  productionDesign: CSFDCreator[];
+}
+
+export interface CSFDTitlesOther {
+  country: string;
+  title: string;
+}
+
+export interface CSFDCreator {
+  /**
+   * CSFD person ID.
+   *
+   * You can always assemble url from ID like this:
+   *
+   * `https://www.csfd.cz/tvurce/${id}`
+   */
+  id: number;
+  name: string;
+  url: string;
+}
+
+export interface CSFDMovieListItem {
+  id: number;
+  title: string;
+  url: string;
+}
+
+export type CSFDGenres =
+  | 'Akční'
+  | 'Animovaný'
+  | 'Dobrodružný'
+  | 'Dokumentární'
+  | 'Drama'
+  | 'Experimentální'
+  | 'Fantasy'
+  | 'Film-Noir'
+  | 'Historický'
+  | 'Horor'
+  | 'Hudební'
+  | 'IMAX'
+  | 'Katastrofický'
+  | 'Komedie'
+  | 'Krátkometrážní'
+  | 'Krimi'
+  | 'Loutkový'
+  | 'Muzikál'
+  | 'Mysteriózní'
+  | 'Naučný'
+  | 'Podobenství'
+  | 'Poetický'
+  | 'Pohádka'
+  | 'Povídkový'
+  | 'Psychologický'
+  | 'Publicistický'
+  | 'Reality-TV'
+  | 'Road movie'
+  | 'Rodinný'
+  | 'Romantický'
+  | 'Sci-Fi'
+  | 'Soutěžní'
+  | 'Sportovní'
+  | 'Stand-up'
+  | 'Talk-show'
+  | 'Taneční'
+  | 'Telenovela'
+  | 'Thriller'
+  | 'Válečný'
+  | 'Western'
+  | 'Zábavný'
+  | 'Životopisný';
+
+export type CSFDCreatorGroups =
+  | 'Režie'
+  | 'Scénář'
+  | 'Kamera'
+  | 'Hudba'
+  | 'Hrají'
+  | 'Produkce'
+  | 'Střih'
+  | 'Předloha'
+  | 'Scénografie'
+  | 'Kostýmy';
+
+export interface CSFDPremiere {
+  country: string;
+  format: string;
+  date: string;
+  company: string;
+}
+
+export type CSFDBoxContent = 'Související' | 'Podobné';
+
+export type CSFDSeasons = CSFDSeasonInfo[]
+
+export interface CSFDSeasonInfo {
+  name: string
+  linkToDetail: string
+  additionalInfo: string
+}

package/src/interfaces/search.interface.ts

@@ -0,0 +1,32 @@
+import { CSFDScreening } from './global';
+import { CSFDCreator } from './movie.interface';
+
+export interface CSFDSearch {
+  movies: CSFDSearchMovie[];
+  tvSeries: CSFDSearchMovie[];
+  creators: CSFDSearchCreator[];
+  users: CSFDSearchUser[];
+}
+
+export interface CSFDSearchMovie extends CSFDScreening {
+  poster: string;
+  origins: string[];
+  creators: CSFDSearchCreators;
+}
+
+export interface CSFDSearchUser {
+  id: number;
+  user: string;
+  userRealName: string;
+  avatar: string;
+  url: string;
+}
+
+export interface CSFDSearchCreator extends CSFDCreator {
+  image: string;
+}
+
+export interface CSFDSearchCreators {
+  directors: CSFDCreator[];
+  actors: CSFDCreator[];
+}

package/src/interfaces/user-ratings.interface.ts

@@ -0,0 +1,21 @@
+import { CSFDFilmTypes, CSFDScreening, CSFDStars } from './global';
+
+export interface CSFDUserRatings extends CSFDScreening {
+  userRating: CSFDStars;
+  userDate: string; // TODO datetime
+}
+
+export interface CSFDUserRatingConfig {
+  includesOnly?: CSFDFilmTypes[];
+  excludes?: CSFDFilmTypes[];
+  /**
+   * Fetch all ratings. (Warning: Use it wisely. Can be detected and banned. Consider using it together with `allPagesDelay` attribute)
+   */
+  allPages?: boolean;
+  /**
+   * Delay on each page request. In milliseconds
+   */
+  allPagesDelay?: number;
+}
+
+export type Colors = 'lightgrey' | 'blue' | 'red' | 'grey';

package/src/services/creator.service.ts

@@ -0,0 +1,34 @@
+import { HTMLElement, parse } from 'node-html-parser';
+import { fetchPage } from '../fetchers';
+import { getBio, getBirthdayInfo, getFilms, getName, getPhoto } from '../helpers/creator.helper';
+import { CSFDCreator } from '../interfaces/creator.interface';
+import { creatorUrl } from '../vars';
+
+export class CreatorScraper {
+  private person: CSFDCreator;
+
+  public async creator(creatorId: number): Promise<CSFDCreator> {
+    const url = creatorUrl(+creatorId);
+    const response = await fetchPage(url);
+
+    const creatorHtml = parse(response);
+
+    const asideNode = creatorHtml.querySelector('.creator-about');
+    const filmsNode = creatorHtml.querySelector('.creator-filmography');
+    this.buildCreator(+creatorId, asideNode, filmsNode);
+    return this.person;
+  }
+
+  private buildCreator(id: number, asideEl: HTMLElement, filmsNode: HTMLElement) {
+    this.person = {
+      id,
+      name: getName(asideEl),
+      birthday: getBirthdayInfo(asideEl)?.birthday,
+      birthplace: getBirthdayInfo(asideEl)?.birthPlace,
+      photo: getPhoto(asideEl),
+      age: getBirthdayInfo(asideEl)?.age || null,
+      bio: getBio(asideEl),
+      films: getFilms(filmsNode)
+    };
+  }
+}

package/src/services/movie.service.ts

@@ -0,0 +1,89 @@
+import { HTMLElement, parse } from 'node-html-parser';
+import { fetchPage } from '../fetchers';
+import {
+  getBoxMovies,
+  getColorRating,
+  getDescriptions,
+  getDuration,
+  getGenres,
+  getGroup,
+  getOrigins,
+  getPoster,
+  getPremieres,
+  getRandomPhoto,
+  getRating,
+  getRatingCount, getSeasonsInfo,
+  getTags,
+  getTitle,
+  getTitlesOther,
+  getTrivia,
+  getType,
+  getVods,
+  getYear
+} from '../helpers/movie.helper';
+import { CSFDFilmTypes } from '../interfaces/global';
+import { CSFDMovie } from '../interfaces/movie.interface';
+import { movieUrl } from '../vars';
+
+export class MovieScraper {
+  private film: CSFDMovie;
+
+  public async movie(movieId: number): Promise<CSFDMovie> {
+    const url = movieUrl(+movieId);
+    const response = await fetchPage(url);
+
+    const movieHtml = parse(response);
+
+    const pageClasses = movieHtml.querySelector('.page-content').classNames.split(' ');
+    const asideNode = movieHtml.querySelector('.aside-movie-profile');
+    const movieNode = movieHtml.querySelector('.main-movie-profile');
+    const jsonLd = movieHtml.querySelector('script[type="application/ld+json"]').innerText;
+    this.buildMovie(+movieId, movieNode, asideNode, pageClasses, jsonLd);
+    return this.film;
+  }
+
+  private buildMovie(
+    movieId: number,
+    el: HTMLElement,
+    asideEl: HTMLElement,
+    pageClasses: string[],
+    jsonLd: string
+  ) {
+    this.film = {
+      id: movieId,
+      title: getTitle(el),
+      year: getYear(jsonLd),
+      duration: getDuration(jsonLd, el),
+      descriptions: getDescriptions(el),
+      genres: getGenres(el),
+      type: getType(el) as CSFDFilmTypes,
+      url: movieUrl(movieId),
+      origins: getOrigins(el),
+      colorRating: getColorRating(pageClasses),
+      rating: getRating(asideEl),
+      ratingCount: getRatingCount(asideEl),
+      titlesOther: getTitlesOther(el),
+      poster: getPoster(el),
+      photo: getRandomPhoto(el),
+      trivia: getTrivia(el),
+      creators: {
+        directors: getGroup(el, 'Režie'),
+        writers: getGroup(el, 'Scénář'),
+        cinematography: getGroup(el, 'Kamera'),
+        music: getGroup(el, 'Hudba'),
+        actors: getGroup(el, 'Hrají'),
+        basedOn: getGroup(el, 'Předloha'),
+        producers: getGroup(el, 'Produkce'),
+        filmEditing: getGroup(el, 'Střih'),
+        costumeDesign: getGroup(el, 'Kostýmy'),
+        productionDesign: getGroup(el, 'Scénografie')
+      },
+      vod: getVods(asideEl),
+      tags: getTags(asideEl),
+      premieres: getPremieres(asideEl),
+      related: getBoxMovies(asideEl, 'Související'),
+      similar: getBoxMovies(asideEl, 'Podobné'),
+      seasons: getSeasonsInfo(el)
+    };
+  }
+}

package/src/services/search.service.ts

@@ -0,0 +1,101 @@
+import { HTMLElement, parse } from 'node-html-parser';
+import { fetchPage } from '../fetchers';
+import { parseIdFromUrl } from '../helpers/global.helper';
+import { getAvatar, getUser, getUserRealName, getUserUrl } from '../helpers/search-user.helper';
+import {
+  getColorRating,
+  getOrigins,
+  getPoster,
+  getTitle,
+  getType,
+  getUrl,
+  getYear,
+  parsePeople
+} from '../helpers/search.helper';
+import { CSFDSearch, CSFDSearchMovie, CSFDSearchUser } from '../interfaces/search.interface';
+import { searchUrl } from '../vars';
+
+export class SearchScraper {
+  public async search(text: string): Promise<CSFDSearch> {
+    const url = searchUrl(text);
+    const response = await fetchPage(url);
+
+    const html = parse(response);
+    const moviesNode = html.querySelectorAll('.main-movies article');
+    const usersNode = html.querySelectorAll('.main-users article');
+    const tvSeriesNode = html.querySelectorAll('.main-series article');
+
+    return this.parseSearch(moviesNode, usersNode, tvSeriesNode);
+  }
+
+  private parseSearch(
+    moviesNode: HTMLElement[],
+    usersNode: HTMLElement[],
+    tvSeriesNode: HTMLElement[]
+  ) {
+    const movies: CSFDSearchMovie[] = [];
+    const users: CSFDSearchUser[] = [];
+    const tvSeries: CSFDSearchMovie[] = [];
+
+    moviesNode.map((m) => {
+      const url = getUrl(m);
+
+      const movie: CSFDSearchMovie = {
+        id: parseIdFromUrl(url),
+        title: getTitle(m),
+        year: getYear(m),
+        url: `https://www.csfd.cz${url}`,
+        type: getType(m),
+        colorRating: getColorRating(m),
+        poster: getPoster(m),
+        origins: getOrigins(m),
+        creators: {
+          directors: parsePeople(m, 'directors'),
+          actors: parsePeople(m, 'actors')
+        }
+      };
+      movies.push(movie);
+    });
+
+    usersNode.map((m) => {
+      const url = getUserUrl(m);
+
+      const user: CSFDSearchUser = {
+        id: parseIdFromUrl(url),
+        user: getUser(m),
+        userRealName: getUserRealName(m),
+        avatar: getAvatar(m),
+        url: `https://www.csfd.cz${url}`
+      };
+      users.push(user);
+    });
+
+    tvSeriesNode.map((m) => {
+      const url = getUrl(m);
+
+      const user: CSFDSearchMovie = {
+        id: parseIdFromUrl(url),
+        title: getTitle(m),
+        year: getYear(m),
+        url: `https://www.csfd.cz${url}`,
+        type: getType(m),
+        colorRating: getColorRating(m),
+        poster: getPoster(m),
+        origins: getOrigins(m),
+        creators: {
+          directors: parsePeople(m, 'directors'),
+          actors: parsePeople(m, 'actors')
+        }
+      };
+      tvSeries.push(user);
+    });
+
+    const search: CSFDSearch = {
+      movies: movies,
+      users: users,
+      tvSeries: tvSeries,
+      creators: []
+    };
+    return search;
+  }
+}

package/src/services/user-ratings.service.ts

@@ -0,0 +1,106 @@
+import { HTMLElement, parse } from 'node-html-parser';
+import { fetchPage } from '../fetchers';
+import {
+  getColorRating,
+  getDate,
+  getId,
+  getTitle,
+  getType,
+  getUrl,
+  getUserRating,
+  getYear,
+  sleep
+} from '../helpers/user-ratings.helper';
+import { CSFDColorRating, CSFDStars } from '../interfaces/global';
+import { CSFDUserRatingConfig, CSFDUserRatings } from '../interfaces/user-ratings.interface';
+import { userRatingsUrl } from '../vars';
+
+export class UserRatingsScraper {
+  private films: CSFDUserRatings[] = [];
+
+  public async userRatings(
+    user: string | number,
+    config?: CSFDUserRatingConfig
+  ): Promise<CSFDUserRatings[]> {
+    let allMovies: CSFDUserRatings[] = [];
+    const url = userRatingsUrl(user);
+    const response = await fetchPage(url);
+
+    const items = parse(response);
+    const movies = items.querySelectorAll('.box-user-rating .table-container tbody tr');
+
+    // Get number of pages
+    const pagesNode = items.querySelector('.pagination');
+    const pages = +pagesNode?.childNodes[pagesNode.childNodes.length - 4].rawText || 1;
+
+    allMovies = this.getPage(config, movies);
+
+    if (config?.allPages) {
+      console.log('User', user, url);
+      console.log('Fetching all pages', pages);
+      for (let i = 2; i <= pages; i++) {
+        console.log('Fetching page', i, 'out of', pages, '...');
+        const url = userRatingsUrl(user, i);
+        const response = await fetchPage(url);
+
+        const items = parse(response);
+        const movies = items.querySelectorAll('.box-user-rating .table-container tbody tr');
+        allMovies = [...this.getPage(config, movies)];
+
+        // Sleep
+        if (config.allPagesDelay) {
+          await sleep(config.allPagesDelay);
+        }
+      }
+      return allMovies;
+    }
+
+    return allMovies;
+  }
+
+  private getPage(config: CSFDUserRatingConfig, movies: HTMLElement[]) {
+    if (config) {
+      if (config.includesOnly?.length && config.excludes?.length) {
+        console.warn(
+          `node-csfd-api:
+        You can not use both parameters 'includesOnly' and 'excludes'.
+        Parameter 'includesOnly' will be used now:`,
+          config.includesOnly
+        );
+      }
+    }
+
+    for (const el of movies) {
+      const type = getType(el);
+
+      // Filtering includesOnly
+      if (config?.includesOnly?.length) {
+        if (config.includesOnly.some((include) => type === include)) {
+          this.buildUserRatings(el);
+        }
+        // Filter exludes
+      } else if (config?.excludes?.length) {
+        if (!config.excludes.some((exclude) => type === exclude)) {
+          this.buildUserRatings(el);
+        }
+      } else {
+        // Without filtering
+        this.buildUserRatings(el);
+      }
+    }
+    return this.films;
+  }
+
+  private buildUserRatings(el: HTMLElement) {
+    this.films.push({
+      id: getId(el),
+      title: getTitle(el),
+      year: getYear(el),
+      type: getType(el),
+      url: getUrl(el),
+      colorRating: getColorRating(el) as CSFDColorRating,
+      userDate: getDate(el),
+      userRating: getUserRating(el) as CSFDStars
+    });
+  }
+}

package/src/vars.ts

@@ -0,0 +1,13 @@
+export const userRatingsUrl = (user: string | number, page?: number): string =>
+  `https://www.csfd.cz/uzivatel/${encodeURIComponent(user)}/hodnoceni/${
+    page ? '?page=' + page : ''
+  }`;
+
+export const movieUrl = (movie: number): string =>
+  `https://www.csfd.cz/film/${encodeURIComponent(movie)}/prehled/`;
+
+export const creatorUrl = (creator: number | string): string =>
+  `https://www.csfd.cz/tvurce/${encodeURIComponent(creator)}`;
+
+export const searchUrl = (text: string): string =>
+  `https://www.csfd.cz/hledat/?q=${encodeURIComponent(text)}`;