metascraper-x 5.49.7 → 5.49.9

package/README.md

@@ -20,17 +20,11 @@ $ npm install metascraper-x --save
 
 #### options
 
-##### resolveUrls
-
-Type: `boolean`
-
-Set to `true` if you want to resolve `t.co` URLs into the final URL.
-
 ##### resolveUrl
 
 Type: `function`
 
-The function implementation that resolves `t.co` into a the final URL.
+The function implementation that resolves URLs (e.g., `t.co`) into the final URL.
 
 ## License
 

package/package.json

@@ -2,7 +2,7 @@
   "name": "metascraper-x",
   "description": "Metascraper integration with x.com",
   "homepage": "https://github.com/microlinkhq/metascraper/packages/metascraper-x",
-  "version": "5.49.7",
+  "version": "5.49.9",
   "types": "src/index.d.ts",
   "main": "src/index.js",
   "author": {
@@ -39,5 +39,5 @@
     "test": "NODE_PATH=.. TZ=UTC ava --timeout 15s"
   },
   "license": "MIT",
-  "gitHead": "4920125394e77b42fcc0636e03fb066b7e1f81ed"
+  "gitHead": "47690490a2eb1f6e3bcd47559fefeaa298c45eaa"
 }

package/src/index.js

@@ -1,28 +1,31 @@
 'use strict'
 
 const {
-  getUrls,
+  $jsonld,
   author,
+  date,
+  description,
+  getUrls,
   image,
   memoizeOne,
   parseUrl,
   title,
   toRule,
-  description,
   url
 } = require('@metascraper/helpers')
 
+const toDescription = toRule(description)
 const toAuthor = toRule(author)
 const toImage = toRule(image)
 const toTitle = toRule(title)
-const toDescription = toRule(description)
+const toDate = toRule(date)
 const toUrl = toRule(url)
 
 const test = memoizeOne(url =>
   ['twitter.com', 'x.com'].includes(parseUrl(url).domain)
 )
 
-module.exports = ({ resolveUrls = false, resolveUrl = url => url } = {}) => {
+module.exports = ({ resolveUrl = url => url } = {}) => {
   const rules = {
     author: [
       toAuthor($ => {
@@ -42,8 +45,10 @@ module.exports = ({ resolveUrls = false, resolveUrl = url => url } = {}) => {
     ],
     description: [
       toDescription(async $ => {
-        let description = $('meta[property="og:description"]').attr('content')
-        if (!resolveUrls) return description
+        let description =
+          $jsonld('mainEntity.description')($) ||
+          $('meta[property="og:description"]').attr('content')
+
         const urls = getUrls(description)
         const resolvedUrls = await Promise.all(urls.map(resolveUrl))
 
@@ -57,13 +62,43 @@ module.exports = ({ resolveUrls = false, resolveUrl = url => url } = {}) => {
     ],
     image: [
       toImage($ => {
-        let imageUrl = $('meta[property="og:image"]').attr('content')
+        let imageUrl =
+          $jsonld('mainEntity.image.contentUrl')($) ||
+          $('meta[property="og:image"]').attr('content')
+
         if (imageUrl?.endsWith('_200x200.jpg')) {
           imageUrl = imageUrl.replace('_200x200.jpg', '_400x400.jpg')
         }
         return imageUrl
       })
     ],
+    date: [
+      toDate(($, url) => {
+        const { pathname } = new URL(url)
+        const parts = pathname.split('/')
+        const username = parts[1].toLowerCase()
+        const isStatus = parts[2] === 'status'
+
+        if (isStatus) {
+          const statusId = parts[3]
+          return (
+            $(`a[href*="/status/${statusId}"] time`).attr('datetime') ||
+            $('article time').attr('datetime') ||
+            $('time').attr('datetime')
+          )
+        }
+
+        return $(`a[href*="/${username}/status/" i] time`)
+          .get()
+          .reduce((acc, el) => {
+            const datetime = $(el).attr('datetime')
+            if (datetime && (!acc || datetime > acc)) {
+              acc = datetime
+            }
+            return acc
+          }, undefined)
+      })
+    ],
     publisher: () => 'X'
   }