html2rss 0.24.0 → 0.26.0

data/schema/html2rss-config.schema.json

@@ -45,12 +45,55 @@
             "null",
             "string"
           ]
+        },
+        "author": {
+          "type": [
+            "null",
+            "string"
+          ]
+        },
+        "image": {
+          "type": [
+            "null",
+            "string"
+          ]
         }
       },
       "required": [
         "url"
       ]
     },
+    "directory": {
+      "type": "object",
+      "properties": {
+        "topics": {
+          "type": "array",
+          "items": {
+            "minLength": 1,
+            "type": "string",
+            "enum": [
+              "sports",
+              "energy",
+              "tech",
+              "science",
+              "news",
+              "entertainment",
+              "jobs",
+              "finance",
+              "security",
+              "travel",
+              "environment",
+              "consumer",
+              "civic",
+              "product",
+              "research"
+            ]
+          },
+          "minItems": 1
+        }
+      },
+      "required": []
+    },
     "headers": {
       "type": "object",
       "description": "HTTP headers applied to every request.",
@@ -94,6 +137,13 @@
       "$schema": "http://json-schema.org/draft-06/schema#",
       "type": "object",
       "properties": {
+        "limit": {
+          "type": "integer",
+          "not": {
+            "type": "null"
+          },
+          "exclusiveMinimum": 0
+        },
         "scraper": {
           "type": "object",
           "properties": {
@@ -182,6 +232,18 @@
               },
               "required": []
             },
+            "xhr_articles": {
+              "type": "object",
+              "properties": {
+                "enabled": {
+                  "type": "boolean",
+                  "not": {
+                    "type": "null"
+                  }
+                }
+              },
+              "required": []
+            },
             "meta_oembed": {
               "type": "object",
               "properties": {
@@ -255,13 +317,6 @@
               "not": {
                 "type": "null"
               }
-            },
-            "min_words_title": {
-              "type": "integer",
-              "not": {
-                "type": "null"
-              },
-              "exclusiveMinimum": 0
             }
           },
           "required": []
@@ -269,6 +324,7 @@
       },
       "required": [],
       "default": {
+        "limit": 25,
         "scraper": {
           "wordpress_api": {
             "enabled": true
@@ -290,6 +346,9 @@
           "json_state": {
             "enabled": true
           },
+          "xhr_articles": {
+            "enabled": true
+          },
           "meta_oembed": {
             "enabled": true
           },
@@ -305,8 +364,7 @@
           }
         },
         "cleanup": {
-          "keep_different_domain": false,
-          "min_words_title": 3
+          "keep_different_domain": false
         }
       }
     },
@@ -402,8 +460,24 @@
           "type": "object",
           "properties": {
             "extractor": {
-              "type": "string",
-              "minLength": 1
+              "description": "Extractor used to pull a value from the selected element.",
+              "oneOf": [
+                {
+                  "$ref": "#/$defs/extractors/attribute"
+                },
+                {
+                  "$ref": "#/$defs/extractors/href"
+                },
+                {
+                  "$ref": "#/$defs/extractors/html"
+                },
+                {
+                  "$ref": "#/$defs/extractors/static"
+                },
+                {
+                  "$ref": "#/$defs/extractors/text"
+                }
+              ]
             },
             "attribute": {
               "type": "string",
@@ -415,8 +489,34 @@
             },
             "post_process": {
               "type": "array",
+              "description": "Ordered transforms applied to the extracted value.",
               "items": {
-                "type": "object"
+                "oneOf": [
+                  {
+                    "$ref": "#/$defs/post_processors/gsub"
+                  },
+                  {
+                    "$ref": "#/$defs/post_processors/html_to_markdown"
+                  },
+                  {
+                    "$ref": "#/$defs/post_processors/markdown_to_html"
+                  },
+                  {
+                    "$ref": "#/$defs/post_processors/parse_time"
+                  },
+                  {
+                    "$ref": "#/$defs/post_processors/parse_uri"
+                  },
+                  {
+                    "$ref": "#/$defs/post_processors/sanitize_html"
+                  },
+                  {
+                    "$ref": "#/$defs/post_processors/substring"
+                  },
+                  {
+                    "$ref": "#/$defs/post_processors/template"
+                  }
+                ]
               }
             },
             "content_type": {
@@ -452,8 +552,24 @@
           "type": "object",
           "properties": {
             "extractor": {
-              "type": "string",
-              "minLength": 1
+              "description": "Extractor used to pull a value from the selected element.",
+              "oneOf": [
+                {
+                  "$ref": "#/$defs/extractors/attribute"
+                },
+                {
+                  "$ref": "#/$defs/extractors/href"
+                },
+                {
+                  "$ref": "#/$defs/extractors/html"
+                },
+                {
+                  "$ref": "#/$defs/extractors/static"
+                },
+                {
+                  "$ref": "#/$defs/extractors/text"
+                }
+              ]
             },
             "attribute": {
               "type": "string",
@@ -465,8 +581,34 @@
             },
             "post_process": {
               "type": "array",
+              "description": "Ordered transforms applied to the extracted value.",
               "items": {
-                "type": "object"
+                "oneOf": [
+                  {
+                    "$ref": "#/$defs/post_processors/gsub"
+                  },
+                  {
+                    "$ref": "#/$defs/post_processors/html_to_markdown"
+                  },
+                  {
+                    "$ref": "#/$defs/post_processors/markdown_to_html"
+                  },
+                  {
+                    "$ref": "#/$defs/post_processors/parse_time"
+                  },
+                  {
+                    "$ref": "#/$defs/post_processors/parse_uri"
+                  },
+                  {
+                    "$ref": "#/$defs/post_processors/sanitize_html"
+                  },
+                  {
+                    "$ref": "#/$defs/post_processors/substring"
+                  },
+                  {
+                    "$ref": "#/$defs/post_processors/template"
+                  }
+                ]
               }
             }
           },
@@ -500,77 +642,18 @@
           },
           "exclusiveMinimum": 0
         },
-        "browserless": {
-          "type": "object",
-          "properties": {
-            "preload": {
-              "type": "object",
-              "properties": {
-                "wait_after_ms": {
-                  "type": "integer",
-                  "not": {
-                    "type": "null"
-                  },
-                  "minimum": 0
-                },
-                "click_selectors": {
-                  "type": "array",
-                  "items": {
-                    "type": "object",
-                    "properties": {
-                      "selector": {
-                        "type": "string",
-                        "minLength": 1
-                      },
-                      "max_clicks": {
-                        "type": "integer",
-                        "not": {
-                          "type": "null"
-                        },
-                        "exclusiveMinimum": 0
-                      },
-                      "wait_after_ms": {
-                        "type": "integer",
-                        "not": {
-                          "type": "null"
-                        },
-                        "minimum": 0
-                      }
-                    },
-                    "required": [
-                      "selector"
-                    ]
-                  }
-                },
-                "scroll_down": {
-                  "type": "object",
-                  "properties": {
-                    "iterations": {
-                      "type": "integer",
-                      "not": {
-                        "type": "null"
-                      },
-                      "exclusiveMinimum": 0
-                    },
-                    "wait_after_ms": {
-                      "type": "integer",
-                      "not": {
-                        "type": "null"
-                      },
-                      "minimum": 0
-                    }
-                  },
-                  "required": []
-                }
-              },
-              "required": []
-            }
-          },
-          "required": []
-        },
         "botasaurus": {
           "type": "object",
           "properties": {
+            "execution_mode": {
+              "type": "string",
+              "minLength": 1,
+              "enum": [
+                "auto",
+                "request",
+                "browser"
+              ]
+            },
             "navigation_mode": {
               "type": "string",
               "minLength": 1,
@@ -578,7 +661,8 @@
                 "auto",
                 "get",
                 "google_get",
-                "google_get_bypass"
+                "google_get_bypass",
+                "organic_get"
               ]
             },
             "max_retries": {
@@ -602,6 +686,18 @@
               },
               "exclusiveMinimum": 0
             },
+            "scroll": {
+              "type": "boolean",
+              "not": {
+                "type": "null"
+              }
+            },
+            "scroll_to_bottom": {
+              "type": "boolean",
+              "not": {
+                "type": "null"
+              }
+            },
             "block_images": {
               "type": "boolean",
               "not": {
@@ -614,6 +710,12 @@
                 "type": "null"
               }
             },
+            "block_trackers": {
+              "type": "boolean",
+              "not": {
+                "type": "null"
+              }
+            },
             "wait_for_complete_page_load": {
               "type": "boolean",
               "not": {
@@ -646,6 +748,12 @@
             "lang": {
               "type": "string",
               "minLength": 1
+            },
+            "cookies": {
+              "type": "object"
+            },
+            "headers": {
+              "type": "object"
             }
           },
           "required": []
@@ -672,5 +780,240 @@
         "auto_source"
       ]
     }
-  ]
+  ],
+  "$defs": {
+    "post_processors": {
+      "gsub": {
+        "type": "object",
+        "title": "gsub",
+        "description": "Replace matches of `pattern` in the extracted string with `replacement` (Ruby String#gsub; pattern may be a regexp-like string).",
+        "examples": [
+          {
+            "name": "gsub",
+            "pattern": "boo",
+            "replacement": "baz"
+          }
+        ],
+        "properties": {
+          "name": {
+            "type": "string",
+            "const": "gsub"
+          },
+          "pattern": {
+            "type": "string"
+          },
+          "replacement": {
+            "type": "string"
+          }
+        },
+        "required": [
+          "name",
+          "pattern",
+          "replacement"
+        ],
+        "additionalProperties": true
+      },
+      "html_to_markdown": {
+        "type": "object",
+        "title": "html_to_markdown",
+        "description": "Sanitize HTML then convert it to Markdown (via ReverseMarkdown).",
+        "examples": [
+          {
+            "name": "html_to_markdown"
+          }
+        ],
+        "properties": {
+          "name": {
+            "type": "string",
+            "const": "html_to_markdown"
+          }
+        },
+        "required": [
+          "name"
+        ],
+        "additionalProperties": true
+      },
+      "markdown_to_html": {
+        "type": "object",
+        "title": "markdown_to_html",
+        "description": "Convert Markdown to HTML (Kramdown) and sanitize the result. Often chained after `template`.",
+        "examples": [
+          {
+            "name": "markdown_to_html"
+          }
+        ],
+        "properties": {
+          "name": {
+            "type": "string",
+            "const": "markdown_to_html"
+          }
+        },
+        "required": [
+          "name"
+        ],
+        "additionalProperties": true
+      },
+      "parse_time": {
+        "type": "object",
+        "title": "parse_time",
+        "description": "Parse a time string with Time.parse and return RFC822, using the channel `time_zone`.",
+        "examples": [
+          {
+            "name": "parse_time"
+          }
+        ],
+        "properties": {
+          "name": {
+            "type": "string",
+            "const": "parse_time"
+          }
+        },
+        "required": [
+          "name"
+        ],
+        "additionalProperties": true
+      },
+      "parse_uri": {
+        "type": "object",
+        "title": "parse_uri",
+        "description": "Normalize a URL string; resolve relative URLs against the channel URL.",
+        "examples": [
+          {
+            "name": "parse_uri"
+          }
+        ],
+        "properties": {
+          "name": {
+            "type": "string",
+            "const": "parse_uri"
+          }
+        },
+        "required": [
+          "name"
+        ],
+        "additionalProperties": true
+      },
+      "sanitize_html": {
+        "type": "object",
+        "title": "sanitize_html",
+        "description": "Sanitize HTML (sanitize gem RELAXED plus html2rss defaults: absolute URLs, safe link/img attributes, wrap lone images in anchors).",
+        "examples": [
+          {
+            "name": "sanitize_html"
+          }
+        ],
+        "properties": {
+          "name": {
+            "type": "string",
+            "const": "sanitize_html"
+          }
+        },
+        "required": [
+          "name"
+        ],
+        "additionalProperties": true
+      },
+      "substring": {
+        "type": "object",
+        "title": "substring",
+        "description": "Return a slice of the extracted string using Integer `start` and optional `end` (Ruby String#[] range semantics; end may be omitted).",
+        "examples": [
+          {
+            "name": "substring",
+            "start": 4,
+            "end": 6
+          }
+        ],
+        "properties": {
+          "name": {
+            "type": "string",
+            "const": "substring"
+          },
+          "start": {
+            "type": "integer"
+          },
+          "end": {
+            "type": "integer"
+          }
+        },
+        "required": [
+          "name",
+          "start"
+        ],
+        "additionalProperties": true
+      },
+      "template": {
+        "type": "object",
+        "title": "template",
+        "description": "Format a string with Kernel#format-style placeholders (`%{key}` / `%<key>s`). `%{self}` is the current selector value; other keys resolve sibling selectors.",
+        "examples": [
+          {
+            "name": "template",
+            "string": "`%{self}` (`%{price}`)"
+          }
+        ],
+        "properties": {
+          "name": {
+            "type": "string",
+            "const": "template"
+          },
+          "string": {
+            "type": "string"
+          }
+        },
+        "required": [
+          "name",
+          "string"
+        ],
+        "additionalProperties": true
+      }
+    },
+    "extractors": {
+      "attribute": {
+        "type": "string",
+        "const": "attribute",
+        "title": "attribute",
+        "description": "Return the value of an HTML attribute on the selected element. Requires sibling selector option `attribute` (attribute name).",
+        "examples": [
+          "attribute"
+        ]
+      },
+      "href": {
+        "type": "string",
+        "const": "href",
+        "title": "href",
+        "description": "Return the absolute URL from the selected element's `href` attribute (relative hrefs are resolved against the channel URL).",
+        "examples": [
+          "href"
+        ]
+      },
+      "html": {
+        "type": "string",
+        "const": "html",
+        "title": "html",
+        "description": "Return the outer HTML of the selected element. Sanitize during post-processing (e.g. `sanitize_html`).",
+        "examples": [
+          "html"
+        ]
+      },
+      "static": {
+        "type": "string",
+        "const": "static",
+        "title": "static",
+        "description": "Return a fixed value from sibling selector option `static` (no DOM read).",
+        "examples": [
+          "static"
+        ]
+      },
+      "text": {
+        "type": "string",
+        "const": "text",
+        "title": "text",
+        "description": "Return collapsed visible text of the selected element (default extractor).",
+        "examples": [
+          "text"
+        ]
+      }
+    }
+  }
 }