indo-scraper 1.0.0

package/src/news/okezone.js

@@ -0,0 +1,72 @@
+const { fetchHTML, cheerio, parseLdJson, parseThumbnail, parseParagraphs, ok, fail } = require('../utils')
+
+/*
+ * Berita terbaru Okezone
+ * @param {object} options - { channel: 'nasional', page: 1, limit: 20 }
+ * channel: nasional | economy | sports | techno | celebrity | lifestyle | otomotif | health
+ */
+const okezone = async (options = {}) => {
+  return new Promise(async (resolve) => {
+    try {
+      const { channel = 'nasional', page = 1, limit = 20 } = options
+      const CHANNELS = {
+        nasional: 'nasional', economy: 'economy', sports: 'sports',
+        techno: 'techno', celebrity: 'celebrity', lifestyle: 'lifestyle',
+        otomotif: 'otomotif', health: 'health',
+      }
+      const sub  = CHANNELS[channel] || 'nasional'
+      const url  = `https://${sub}.okezone.com/indeks?page=${page}`
+      const html = await fetchHTML(url, { Referer: 'https://www.okezone.com' })
+      const $ = cheerio.load(html)
+      const articles = []
+
+      // Container: parent div yang berisi .box-text
+      $('div.box-text').each((i, el) => {
+        if (articles.length >= limit) return false
+        const $el    = $(el)
+        const linkEl = $el.find('a.title').first()
+        const title  = linkEl.text().trim()
+        const href   = linkEl.attr('href') || ''
+        if (!title || !href) return
+        const container = $el.parent()
+        const imgEl  = container.find('img').first()
+        articles.push({
+          title,
+          url: href.startsWith('http') ? href : `https://${sub}.okezone.com${href}`,
+          image: imgEl.attr('data-src') || imgEl.attr('src') || null,
+          category: $el.find('a.kanal').text().trim() || channel,
+          date: $el.find('div.timego').text().trim() || null,
+          source: 'okezone',
+        })
+      })
+      if (!articles.length) return resolve(fail('Data tidak ditemukan'))
+      resolve(ok(articles))
+    } catch (e) { console.log(e); resolve(fail(e)) }
+  })
+}
+
+/*
+ * Artikel lengkap Okezone
+ * @param {string} url
+ */
+const okenewsArticle = async (url) => {
+  return new Promise(async (resolve) => {
+    try {
+      const html = await fetchHTML(url, { Referer: 'https://www.okezone.com' })
+      const $ = cheerio.load(html)
+      const ld = parseLdJson($, html)
+      const paragraphs = parseParagraphs($, '.description.read p')
+      resolve(ok({
+        title: ld?.headline || $('h1.title-content').first().text().trim(),
+        author: ld?.author?.name || $('.author, .nm-reporter').first().text().trim(),
+        date: ld?.datePublished || html.match(/"datePublished"\s*:\s*"([^"]+)"/)?.[ 1] || '',
+        category: $('.category').first().text().trim() || ld?.articleSection || '',
+        description: ld?.description || html.match(/"description"\s*:\s*"([^"]+)"/)?.[ 1] || '',
+        thumbnail: parseThumbnail(ld),
+        content: paragraphs.join('\n\n'), paragraphs,
+      }))
+    } catch (e) { console.log(e); resolve(fail(e)) }
+  })
+}
+
+module.exports = { okezone, okenewsArticle }

package/src/news/republika.js

@@ -0,0 +1,73 @@
+const { fetchHTML, cheerio, parseLdJson, parseThumbnail, parseParagraphs, ok, fail } = require('../utils')
+
+/*
+ * Berita terbaru Republika
+ * @param {object} options - { channel: 'nasional', page: 1, limit: 20 }
+ * channel: nasional | internasional | ekonomi | olahraga | hiburan | tekno | gaya_hidup
+ */
+const republika = async (options = {}) => {
+  return new Promise(async (resolve) => {
+    try {
+      const { channel = 'nasional', page = 1, limit = 20 } = options
+      const ch  = channel === 'gaya_hidup' ? 'gaya-hidup' : channel
+      const url = `https://republika.co.id/berita/${ch}?page=${page}`
+      const html = await fetchHTML(url, { Referer: 'https://republika.co.id' })
+      const $ = cheerio.load(html)
+      const articles = []
+      $('.link-mobile-headline').each((i, el) => {
+        if (articles.length >= limit) return false
+        const $el   = $(el)
+        const href  = $el.attr('href') || ''
+        const title = $el.find('h1.card-text, h2.card-text, .card-text').first().text().trim()
+        if (!title || !href) return
+        const image    = $el.find('img.lazy').first().attr('data-original') || null
+        const smalls   = $el.find('small')
+        const category = smalls.filter('.text-primary').first().text().trim() || channel
+        const date     = smalls.not('.text-primary').first().text().replace(/^-\s*/, '').trim() || null
+        articles.push({
+          title,
+          url: href.startsWith('http') ? href : `https://republika.co.id${href}`,
+          image, category, date,
+          source: 'republika',
+        })
+      })
+      if (!articles.length) return resolve(fail('Data tidak ditemukan'))
+      resolve(ok(articles))
+    } catch (e) { console.log(e); resolve(fail(e)) }
+  })
+}
+
+/*
+ * Artikel lengkap Republika
+ * @param {string} url
+ */
+const republikaArticle = async (url) => {
+  return new Promise(async (resolve) => {
+    try {
+      const html = await fetchHTML(url, { Referer: 'https://republika.co.id' })
+      const $ = cheerio.load(html)
+      const ld = parseLdJson($, html)
+
+      // Ekstrak kategori dari subdomain URL (misal khazanah.republika.co.id → 'khazanah')
+      let category = ''
+      try {
+        const sub = new URL(url).hostname.split('.')[0]
+        if (sub !== 'republika' && sub !== 'www') category = sub
+      } catch (_) {}
+      category = category || ld?.articleSection || ''
+
+      const paragraphs = parseParagraphs($, 'p.paragraphx')
+      resolve(ok({
+        title: ld?.headline || $('h1').first().text().trim(),
+        author: ld?.author?.name || $('.write-by').first().text().replace(/^Oleh:\s*/i, '').trim(),
+        date: ld?.datePublished || html.match(/"datePublished"\s*:\s*"([^"]+)"/)?.[ 1] || '',
+        category,
+        description: ld?.description || html.match(/"description"\s*:\s*"([^"]+)"/)?.[ 1] || '',
+        thumbnail: parseThumbnail(ld),
+        content: paragraphs.join('\n\n'), paragraphs,
+      }))
+    } catch (e) { console.log(e); resolve(fail(e)) }
+  })
+}
+
+module.exports = { republika, republikaArticle }

package/src/news/tribun.js

@@ -0,0 +1,95 @@
+const { fetchHTML, cheerio, parseLdJson, parseThumbnail, parseParagraphs, ok, fail } = require('../utils')
+
+// Coba ambil URL gambar dari berbagai atribut lazy-load
+const pickImg = ($, el) => {
+  const attrs = ['data-src', 'data-original', 'data-lazy', 'data-image', 'src']
+  for (const a of attrs) {
+    const v = $(el).attr(a) || ''
+    if (v && !v.startsWith('data:') && v.startsWith('http')) return v
+  }
+  const ss = $(el).attr('data-srcset') || $(el).attr('srcset') || ''
+  if (ss) return ss.trim().split(/[\s,]+/).find(s => s.startsWith('http')) || null
+  return null
+}
+
+/*
+ * Berita terbaru Tribunnews
+ * @param {object} options - { channel: 'nasional', page: 1, limit: 20 }
+ * channel: nasional | regional | internasional | sport | bisnis | seleb | lifestyle | techno | otomotif
+ */
+const tribun = async (options = {}) => {
+  return new Promise(async (resolve) => {
+    try {
+      const { channel = 'nasional', page = 1, limit = 20 } = options
+      const url  = `https://www.tribunnews.com/${channel}?page=${page}`
+      const html = await fetchHTML(url, { Referer: 'https://www.tribunnews.com' })
+      const $ = cheerio.load(html)
+      const articles = []
+      $("h3 a[href*='tribunnews.com']").each((i, el) => {
+        if (articles.length >= limit) return false
+        const $el  = $(el)
+        const title = $el.attr('title') || $el.text().trim()
+        const href  = $el.attr('href') || ''
+        if (!title || !href) return
+
+        // Naik ke container terluas: coba li/article dulu, fallback ke parent div
+        let container = $el.closest('li, article')
+        if (!container.length) container = $el.closest('div').parent()
+
+        // Cari img di seluruh container — coba semua atribut lazy-load
+        let image = null
+        container.find('img').each((_, img) => {
+          if (image) return false
+          image = pickImg($, img) || null
+        })
+
+        articles.push({
+          title, url: href, image,
+          category: container.find('h4 a.tsa-2').text().trim() || channel,
+          date: container.find('time span').text().trim() || null,
+          source: 'tribunnews',
+        })
+      })
+      if (!articles.length) return resolve(fail('Data tidak ditemukan'))
+      resolve(ok(articles))
+    } catch (e) { console.log(e); resolve(fail(e)) }
+  })
+}
+
+/*
+ * Artikel lengkap Tribunnews
+ * @param {string} url
+ */
+const tribunArticle = async (url) => {
+  return new Promise(async (resolve) => {
+    try {
+      const html = await fetchHTML(url, { Referer: 'https://www.tribunnews.com' })
+      const $ = cheerio.load(html)
+      const ld = parseLdJson($, html)
+      let author = ''
+      if (ld?.author) author = Array.isArray(ld.author) ? ld.author[0]?.name || '' : ld.author.name || ''
+      if (!author) author = html.match(/'penulis'\s*:\s*'([^']+)'/)?.[1] || ''
+
+      // 1) ld+json  2) og:image (server-side, tidak lazy-load)  3) DOM fallback
+      let thumbnail = parseThumbnail(ld)
+      if (!thumbnail) thumbnail = $('meta[property="og:image"]').attr('content') || null
+      if (!thumbnail) {
+        const imgEl = $('.img-holder img, .wrap_img img, figure img, #article-body img, .photo img').first()
+        if (imgEl.length) thumbnail = pickImg($, imgEl) || null
+      }
+
+      const paragraphs = parseParagraphs($, 'div#article-body p, div.txt-article p')
+      resolve(ok({
+        title: ld?.headline || $('h1').first().text().trim(),
+        author,
+        date: ld?.datePublished || html.match(/"datePublished"\s*:\s*"([^"]+)"/)?.[1] || '',
+        category: ld?.articleSection || $('.breadcrumb a').eq(1).text().trim(),
+        description: ld?.description || html.match(/"description"\s*:\s*"([^"]+)"/)?.[1] || '',
+        thumbnail,
+        content: paragraphs.join('\n\n'), paragraphs,
+      }))
+    } catch (e) { console.log(e); resolve(fail(e)) }
+  })
+}
+
+module.exports = { tribun, tribunArticle }

package/src/tools/simsimi.js

@@ -0,0 +1,69 @@
+const { axios, ok, fail } = require('../utils')
+
+/*
+ * SimSimi / AI Chat
+ * Source: widipe.com
+ * @param {string} text
+ */
+
+const simsimi = async (text) => {
+  return new Promise(async (resolve) => {
+    try {
+
+      if (!text)
+        return resolve(
+          fail('Text tidak boleh kosong')
+        )
+
+      const res = await axios.get(
+        'https://widipe.com/simi',
+        {
+          params: {
+            text
+          },
+          headers: {
+            'User-Agent':
+              'Mozilla/5.0 (Linux; Android 13; Pixel 7)',
+
+            'Accept':
+              'application/json'
+          },
+          timeout: 15000
+        }
+      )
+
+      const data = res.data
+
+      /*
+       * parse response
+       */
+
+      const answer =
+        data.result ||
+        data.message ||
+        data.response
+
+      if (!answer)
+        return resolve(
+          fail('Tidak ada respon')
+        )
+
+      resolve(ok({
+        question: text,
+        answer
+      }))
+
+    } catch (e) {
+
+      console.log(e)
+
+      resolve(fail(
+        e?.response?.data ||
+        e?.response?.status ||
+        e.message
+      ))
+    }
+  })
+}
+
+module.exports = { simsimi }

package/src/tools/ssweb.js

@@ -0,0 +1,35 @@
+const { ok, fail } = require('../utils')
+
+const DEVICES = {
+  mobile:  { width: 390,  height: 844,  mobile: true  },
+  tablet:  { width: 768,  height: 1024, mobile: true  },
+  desktop: { width: 1920, height: 1080, mobile: false },
+  hd:      { width: 2560, height: 1440, mobile: false },
+  '4k':    { width: 3840, height: 2160, mobile: false },
+}
+
+/*
+ * Website Screenshot via Microlink API
+ * @param {string} url
+ * @param {string} device - mobile | tablet | desktop | hd | 4k
+ */
+const ssweb = async (url, device = 'desktop') => {
+  return new Promise(async (resolve) => {
+    try {
+      if (!url) return resolve(fail('URL tidak boleh kosong'))
+      if (!/^https?:\/\//.test(url)) url = 'https://' + url
+
+      const { width, height, mobile } = DEVICES[device.toLowerCase()] || DEVICES.desktop
+
+      const screenshot = 'https://api.microlink.io/?' + new URLSearchParams({
+        url, screenshot: 'true', meta: 'false', embed: 'screenshot.url',
+        viewport: `${width}x${height}`, deviceScaleFactor: '1',
+        mobile: String(mobile), fullPage: 'true',
+      })
+
+      resolve(ok({ url, device, resolution: `${width}x${height}`, screenshot }))
+    } catch (e) { console.log(e); resolve(fail(e)) }
+  })
+}
+
+module.exports = { ssweb }

package/src/utils.js

@@ -0,0 +1,79 @@
+const axios   = require('axios')
+const cheerio = require('cheerio')
+
+// ── Headers ───────────────────────────────────────────────────────────────────
+
+const HEADERS = {
+  'User-Agent': 'Mozilla/5.0 (Linux; Android 13; Pixel 7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.6367.82 Mobile Safari/537.36',
+  'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8',
+  'Accept-Language': 'id-ID,id;q=0.9,en-US;q=0.8',
+  'Accept-Encoding': 'gzip, deflate, br',
+  'Sec-Fetch-Dest': 'document',
+  'Sec-Fetch-Mode': 'navigate',
+  'Sec-Fetch-Site': 'none',
+  'Sec-Fetch-User': '?1',
+  'Upgrade-Insecure-Requests': '1',
+}
+
+async function fetchHTML(url, headers = {}) {
+  let last
+  for (let i = 0; i <= 2; i++) {
+    try {
+      const res = await axios.get(url, { headers: { ...HEADERS, ...headers }, timeout: 10000, responseType: 'text' })
+      return res.data
+    } catch (e) {
+      last = e
+      await new Promise(r => setTimeout(r, 500 * (i + 1)))
+    }
+  }
+  throw new Error(`Gagal fetch ${url}: ${last.message}`)
+}
+
+async function fetchJSON(url, headers = {}) {
+  const res = await axios.get(url, {
+    headers: {
+      'User-Agent': 'Mozilla/5.0 (Linux; Android 13) AppleWebKit/537.36 Chrome/124.0 Mobile Safari/537.36',
+      'Accept': 'application/json, text/plain, */*',
+      'Accept-Language': 'id-ID,id;q=0.9',
+      'Referer': 'https://www.bmkg.go.id/',
+      'Origin': 'https://www.bmkg.go.id',
+      ...headers
+    },
+    timeout: 10000
+  })
+  return res.data
+}
+
+function parseLdJson($, html) {
+  let ld = null
+  $('script[type="application/ld+json"]').each((_, el) => {
+    try { const j = JSON.parse($(el).html()); if (j.datePublished) ld = j } catch (_) {}
+  })
+  return ld
+}
+
+function parseThumbnail(ld) {
+  if (!ld?.image) return null
+  if (typeof ld.image === 'string') return ld.image
+  if (ld.image.url) return ld.image.url
+  if (Array.isArray(ld.image)) return ld.image[0]?.url || ld.image[0] || null
+  return null
+}
+
+function parseParagraphs($, selector) {
+  const SKIP  = /^(baca juga|simak juga|artikel terkait|lihat juga|advertisement|iklan)/i
+  const STRIP = /baca juga\s*:.*?(?=\n|$)/gi
+  const result = []
+  $(selector).each((_, el) => {
+    let text = $(el).text().trim().replace(STRIP, '').trim()
+    if (text.length > 30 && !SKIP.test(text)) result.push(text)
+  })
+  return result
+}
+
+// ── Responses ─────────────────────────────────────────────────────────────────
+
+const ok   = (data) => ({ creator: global.creator, status: true,  data })
+const fail = (e)    => ({ creator: global.creator, status: false, msg: e instanceof Error ? e.message : String(e) })
+
+module.exports = { axios, cheerio, HEADERS, fetchHTML, fetchJSON, parseLdJson, parseThumbnail, parseParagraphs, ok, fail }