RubyGems - html2rss - Versions diffs - 0.17.0 → 0.19.0 - Mend

html2rss 0.17.0 → 0.19.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (127) hide show

checksums.yaml +4 -4
data/README.md +90 -639
data/exe/html2rss +1 -1
data/html2rss.gemspec +5 -2
data/lib/html2rss/articles/deduplicator.rb +50 -0
data/lib/html2rss/auto_source/cleanup.rb +44 -5
data/lib/html2rss/auto_source/scraper/html.rb +123 -43
data/lib/html2rss/auto_source/scraper/json_state.rb +457 -0
data/lib/html2rss/auto_source/scraper/microdata.rb +505 -0
data/lib/html2rss/auto_source/scraper/schema/category_extractor.rb +102 -0
data/lib/html2rss/auto_source/scraper/schema/item_list.rb +3 -2
data/lib/html2rss/auto_source/scraper/schema/list_item.rb +6 -4
data/lib/html2rss/auto_source/scraper/schema/thing.rb +69 -8
data/lib/html2rss/auto_source/scraper/schema.rb +27 -12
data/lib/html2rss/auto_source/scraper/semantic_html/anchor_selector.rb +204 -0
data/lib/html2rss/auto_source/scraper/semantic_html.rb +88 -79
data/lib/html2rss/auto_source/scraper/wordpress_api/page_scope.rb +311 -0
data/lib/html2rss/auto_source/scraper/wordpress_api/posts_endpoint.rb +135 -0
data/lib/html2rss/auto_source/scraper/wordpress_api.rb +186 -0
data/lib/html2rss/auto_source/scraper.rb +160 -8
data/lib/html2rss/auto_source.rb +123 -47
data/lib/html2rss/blocked_surface.rb +65 -0
data/lib/html2rss/category_extractor.rb +82 -0
data/lib/html2rss/cli.rb +194 -23
data/lib/html2rss/config/class_methods.rb +178 -0
data/lib/html2rss/config/dynamic_params.rb +70 -0
data/lib/html2rss/config/multiple_feeds_config.rb +51 -0
data/lib/html2rss/config/request_headers.rb +136 -0
data/lib/html2rss/config/schema.rb +240 -0
data/lib/html2rss/config/validator.rb +146 -0
data/lib/html2rss/config.rb +118 -61
data/lib/html2rss/error.rb +31 -0
data/lib/html2rss/feed_pipeline/auto_fallback.rb +127 -0
data/lib/html2rss/feed_pipeline.rb +127 -0
data/lib/html2rss/hash_util.rb +101 -0
data/lib/html2rss/html_extractor/date_extractor.rb +20 -0
data/lib/html2rss/html_extractor/enclosure_extractor.rb +120 -0
data/lib/html2rss/html_extractor/image_extractor.rb +58 -0
data/lib/html2rss/html_extractor.rb +141 -0
data/lib/html2rss/html_navigator.rb +54 -0
data/lib/html2rss/json_feed_builder/item.rb +94 -0
data/lib/html2rss/json_feed_builder.rb +59 -0
data/lib/html2rss/rendering/audio_renderer.rb +36 -0
data/lib/html2rss/rendering/description_builder.rb +87 -0
data/lib/html2rss/rendering/image_renderer.rb +41 -0
data/lib/html2rss/rendering/media_renderer.rb +37 -0
data/lib/html2rss/rendering/pdf_renderer.rb +34 -0
data/lib/html2rss/rendering/video_renderer.rb +36 -0
data/lib/html2rss/rendering.rb +23 -0
data/lib/html2rss/request_controls.rb +123 -0
data/lib/html2rss/request_service/botasaurus_contract.rb +161 -0
data/lib/html2rss/request_service/botasaurus_strategy.rb +98 -0
data/lib/html2rss/request_service/browserless_strategy.rb +103 -7
data/lib/html2rss/request_service/budget.rb +39 -0
data/lib/html2rss/request_service/context.rb +77 -21
data/lib/html2rss/request_service/faraday_strategy.rb +137 -5
data/lib/html2rss/request_service/policy.rb +252 -0
data/lib/html2rss/request_service/puppet_commander.rb +212 -13
data/lib/html2rss/request_service/response.rb +51 -3
data/lib/html2rss/request_service/response_guard.rb +62 -0
data/lib/html2rss/request_service.rb +50 -15
data/lib/html2rss/request_session/rel_next_pager.rb +70 -0
data/lib/html2rss/request_session/runtime_input.rb +71 -0
data/lib/html2rss/request_session/runtime_policy.rb +83 -0
data/lib/html2rss/request_session.rb +122 -0
data/lib/html2rss/rss_builder/article.rb +187 -0
data/lib/html2rss/rss_builder/channel.rb +105 -11
data/lib/html2rss/rss_builder/enclosure.rb +62 -0
data/lib/html2rss/rss_builder/stylesheet.rb +8 -4
data/lib/html2rss/rss_builder.rb +76 -71
data/lib/html2rss/selectors/config.rb +123 -0
data/lib/html2rss/selectors/extractors/attribute.rb +52 -0
data/lib/html2rss/selectors/extractors/href.rb +55 -0
data/lib/html2rss/selectors/extractors/html.rb +49 -0
data/lib/html2rss/selectors/extractors/static.rb +42 -0
data/lib/html2rss/selectors/extractors/text.rb +47 -0
data/lib/html2rss/selectors/extractors.rb +53 -0
data/lib/html2rss/selectors/object_to_xml_converter.rb +62 -0
data/lib/html2rss/selectors/post_processors/base.rb +80 -0
data/lib/html2rss/selectors/post_processors/gsub.rb +88 -0
data/lib/html2rss/selectors/post_processors/html_to_markdown.rb +48 -0
data/lib/html2rss/selectors/post_processors/html_transformers/transform_urls_to_absolute_ones.rb +44 -0
data/lib/html2rss/selectors/post_processors/html_transformers/wrap_img_in_a.rb +53 -0
data/lib/html2rss/selectors/post_processors/markdown_to_html.rb +55 -0
data/lib/html2rss/selectors/post_processors/parse_time.rb +78 -0
data/lib/html2rss/selectors/post_processors/parse_uri.rb +43 -0
data/lib/html2rss/selectors/post_processors/sanitize_html.rb +154 -0
data/lib/html2rss/selectors/post_processors/substring.rb +77 -0
data/lib/html2rss/selectors/post_processors/template.rb +76 -0
data/lib/html2rss/selectors/post_processors.rb +48 -0
data/lib/html2rss/selectors.rb +301 -0
data/lib/html2rss/url.rb +266 -0
data/lib/html2rss/version.rb +2 -1
data/lib/html2rss.rb +67 -71
data/lib/tasks/config_schema.rake +17 -0
data/schema/html2rss-config.schema.json +551 -0
metadata +120 -38
data/lib/html2rss/attribute_post_processors/base.rb +0 -74
data/lib/html2rss/attribute_post_processors/gsub.rb +0 -64
data/lib/html2rss/attribute_post_processors/html_to_markdown.rb +0 -43
data/lib/html2rss/attribute_post_processors/html_transformers/transform_urls_to_absolute_ones.rb +0 -27
data/lib/html2rss/attribute_post_processors/html_transformers/wrap_img_in_a.rb +0 -41
data/lib/html2rss/attribute_post_processors/markdown_to_html.rb +0 -50
data/lib/html2rss/attribute_post_processors/parse_time.rb +0 -46
data/lib/html2rss/attribute_post_processors/parse_uri.rb +0 -46
data/lib/html2rss/attribute_post_processors/sanitize_html.rb +0 -115
data/lib/html2rss/attribute_post_processors/substring.rb +0 -72
data/lib/html2rss/attribute_post_processors/template.rb +0 -101
data/lib/html2rss/attribute_post_processors.rb +0 -44
data/lib/html2rss/auto_source/article.rb +0 -127
data/lib/html2rss/auto_source/channel.rb +0 -78
data/lib/html2rss/auto_source/reducer.rb +0 -48
data/lib/html2rss/auto_source/rss_builder.rb +0 -70
data/lib/html2rss/auto_source/scraper/semantic_html/extractor.rb +0 -136
data/lib/html2rss/auto_source/scraper/semantic_html/image.rb +0 -54
data/lib/html2rss/config/channel.rb +0 -125
data/lib/html2rss/config/selectors.rb +0 -103
data/lib/html2rss/item.rb +0 -186
data/lib/html2rss/item_extractors/attribute.rb +0 -50
data/lib/html2rss/item_extractors/href.rb +0 -52
data/lib/html2rss/item_extractors/html.rb +0 -46
data/lib/html2rss/item_extractors/static.rb +0 -39
data/lib/html2rss/item_extractors/text.rb +0 -44
data/lib/html2rss/item_extractors.rb +0 -88
data/lib/html2rss/object_to_xml_converter.rb +0 -56
data/lib/html2rss/rss_builder/item.rb +0 -83
data/lib/html2rss/utils.rb +0 -113

data/schema/html2rss-config.schema.json ADDED Viewed

@@ -0,0 +1,551 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "type": "object",
+  "properties": {
+    "strategy": {
+      "type": "string",
+      "not": {
+        "type": "null"
+      }
+    },
+    "channel": {
+      "type": "object",
+      "properties": {
+        "url": {
+          "type": "string",
+          "minLength": 1
+        },
+        "title": {
+          "type": [
+            "null",
+            "string"
+          ]
+        },
+        "description": {
+          "type": [
+            "null",
+            "string"
+          ]
+        },
+        "language": {
+          "type": [
+            "null",
+            "string"
+          ]
+        },
+        "ttl": {
+          "type": [
+            "null",
+            "integer"
+          ],
+          "exclusiveMinimum": 0
+        },
+        "time_zone": {
+          "type": [
+            "null",
+            "string"
+          ]
+        }
+      },
+      "required": [
+        "url"
+      ]
+    },
+    "headers": {
+      "type": "object",
+      "description": "HTTP headers applied to every request.",
+      "additionalProperties": {
+        "type": "string"
+      }
+    },
+    "stylesheets": {
+      "type": "array",
+      "description": "Collection of stylesheets to attach to the RSS feed.",
+      "items": {
+        "$schema": "http://json-schema.org/draft-06/schema#",
+        "type": "object",
+        "properties": {
+          "href": {
+            "type": "string",
+            "minLength": 1
+          },
+          "type": {
+            "type": "string",
+            "minLength": 1,
+            "enum": [
+              "text/css",
+              "text/xsl"
+            ]
+          },
+          "media": {
+            "type": [
+              "null",
+              "string"
+            ]
+          }
+        },
+        "required": [
+          "href",
+          "type"
+        ]
+      }
+    },
+    "auto_source": {
+      "$schema": "http://json-schema.org/draft-06/schema#",
+      "type": "object",
+      "properties": {
+        "scraper": {
+          "type": "object",
+          "properties": {
+            "wordpress_api": {
+              "type": "object",
+              "properties": {
+                "enabled": {
+                  "type": "boolean",
+                  "not": {
+                    "type": "null"
+                  }
+                }
+              },
+              "required": []
+            },
+            "schema": {
+              "type": "object",
+              "properties": {
+                "enabled": {
+                  "type": "boolean",
+                  "not": {
+                    "type": "null"
+                  }
+                }
+              },
+              "required": []
+            },
+            "microdata": {
+              "type": "object",
+              "properties": {
+                "enabled": {
+                  "type": "boolean",
+                  "not": {
+                    "type": "null"
+                  }
+                }
+              },
+              "required": []
+            },
+            "json_state": {
+              "type": "object",
+              "properties": {
+                "enabled": {
+                  "type": "boolean",
+                  "not": {
+                    "type": "null"
+                  }
+                }
+              },
+              "required": []
+            },
+            "semantic_html": {
+              "type": "object",
+              "properties": {
+                "enabled": {
+                  "type": "boolean",
+                  "not": {
+                    "type": "null"
+                  }
+                }
+              },
+              "required": []
+            },
+            "html": {
+              "type": "object",
+              "properties": {
+                "enabled": {
+                  "type": "boolean",
+                  "not": {
+                    "type": "null"
+                  }
+                },
+                "minimum_selector_frequency": {
+                  "type": "integer",
+                  "not": {
+                    "type": "null"
+                  },
+                  "exclusiveMinimum": 0
+                },
+                "use_top_selectors": {
+                  "type": "integer",
+                  "not": {
+                    "type": "null"
+                  },
+                  "exclusiveMinimum": 0
+                }
+              },
+              "required": []
+            }
+          },
+          "required": []
+        },
+        "cleanup": {
+          "type": "object",
+          "properties": {
+            "keep_different_domain": {
+              "type": "boolean",
+              "not": {
+                "type": "null"
+              }
+            },
+            "min_words_title": {
+              "type": "integer",
+              "not": {
+                "type": "null"
+              },
+              "exclusiveMinimum": 0
+            }
+          },
+          "required": []
+        }
+      },
+      "required": [],
+      "default": {
+        "scraper": {
+          "wordpress_api": {
+            "enabled": true
+          },
+          "schema": {
+            "enabled": true
+          },
+          "microdata": {
+            "enabled": true
+          },
+          "json_state": {
+            "enabled": true
+          },
+          "semantic_html": {
+            "enabled": true
+          },
+          "html": {
+            "enabled": true,
+            "minimum_selector_frequency": 2,
+            "use_top_selectors": 5
+          }
+        },
+        "cleanup": {
+          "keep_different_domain": false,
+          "min_words_title": 3
+        }
+      }
+    },
+    "selectors": {
+      "type": "object",
+      "description": "Selectors used to extract article attributes.",
+      "properties": {
+        "items": {
+          "$schema": "http://json-schema.org/draft-06/schema#",
+          "type": "object",
+          "properties": {
+            "selector": {
+              "type": "string",
+              "minLength": 1
+            },
+            "order": {
+              "enum": [
+                "reverse"
+              ],
+              "not": {
+                "type": "null"
+              }
+            },
+            "enhance": {
+              "type": "boolean",
+              "not": {
+                "type": "null"
+              }
+            },
+            "pagination": {
+              "type": "object",
+              "properties": {
+                "max_pages": {
+                  "type": "integer",
+                  "not": {
+                    "type": "null"
+                  },
+                  "exclusiveMinimum": 0
+                }
+              },
+              "required": [
+                "max_pages"
+              ]
+            }
+          },
+          "required": [
+            "selector"
+          ],
+          "description": "Defines the items selector and optional enhancement settings."
+        },
+        "enclosure": {
+          "$schema": "http://json-schema.org/draft-06/schema#",
+          "type": "object",
+          "properties": {
+            "extractor": {
+              "type": "string",
+              "minLength": 1
+            },
+            "attribute": {
+              "type": "string",
+              "minLength": 1
+            },
+            "static": {
+              "type": "string",
+              "minLength": 1
+            },
+            "post_process": {
+              "type": "array",
+              "items": {
+                "type": "object"
+              }
+            },
+            "content_type": {
+              "type": "string",
+              "minLength": 1
+            }
+          },
+          "required": [],
+          "description": "Describes enclosure extraction settings."
+        },
+        "guid": {
+          "type": "array",
+          "description": "List of selector keys used to build the GUID. Each entry must reference a sibling selector key; runtime validation enforces those references.",
+          "minItems": 1,
+          "items": {
+            "type": "string",
+            "description": "Selector key defined elsewhere in this object."
+          }
+        },
+        "categories": {
+          "type": "array",
+          "description": "List of selector keys whose values will be used as categories. Each entry must reference a sibling selector key; runtime validation enforces those references.",
+          "minItems": 1,
+          "items": {
+            "type": "string",
+            "description": "Selector key defined elsewhere in this object."
+          }
+        }
+      },
+      "patternProperties": {
+        "^(?!items$|enclosure$|guid$|categories$).+$": {
+          "$schema": "http://json-schema.org/draft-06/schema#",
+          "type": "object",
+          "properties": {
+            "extractor": {
+              "type": "string",
+              "minLength": 1
+            },
+            "attribute": {
+              "type": "string",
+              "minLength": 1
+            },
+            "static": {
+              "type": "string",
+              "minLength": 1
+            },
+            "post_process": {
+              "type": "array",
+              "items": {
+                "type": "object"
+              }
+            }
+          },
+          "required": [],
+          "description": "Dynamic selector definition keyed by attribute name."
+        }
+      },
+      "additionalProperties": true
+    },
+    "request": {
+      "type": "object",
+      "properties": {
+        "max_redirects": {
+          "type": "integer",
+          "not": {
+            "type": "null"
+          },
+          "minimum": 0
+        },
+        "max_requests": {
+          "type": "integer",
+          "not": {
+            "type": "null"
+          },
+          "exclusiveMinimum": 0
+        },
+        "browserless": {
+          "type": "object",
+          "properties": {
+            "preload": {
+              "type": "object",
+              "properties": {
+                "wait_after_ms": {
+                  "type": "integer",
+                  "not": {
+                    "type": "null"
+                  },
+                  "minimum": 0
+                },
+                "click_selectors": {
+                  "type": "array",
+                  "items": {
+                    "type": "object",
+                    "properties": {
+                      "selector": {
+                        "type": "string",
+                        "minLength": 1
+                      },
+                      "max_clicks": {
+                        "type": "integer",
+                        "not": {
+                          "type": "null"
+                        },
+                        "exclusiveMinimum": 0
+                      },
+                      "wait_after_ms": {
+                        "type": "integer",
+                        "not": {
+                          "type": "null"
+                        },
+                        "minimum": 0
+                      }
+                    },
+                    "required": [
+                      "selector"
+                    ]
+                  }
+                },
+                "scroll_down": {
+                  "type": "object",
+                  "properties": {
+                    "iterations": {
+                      "type": "integer",
+                      "not": {
+                        "type": "null"
+                      },
+                      "exclusiveMinimum": 0
+                    },
+                    "wait_after_ms": {
+                      "type": "integer",
+                      "not": {
+                        "type": "null"
+                      },
+                      "minimum": 0
+                    }
+                  },
+                  "required": []
+                }
+              },
+              "required": []
+            }
+          },
+          "required": []
+        },
+        "botasaurus": {
+          "type": "object",
+          "properties": {
+            "navigation_mode": {
+              "type": "string",
+              "minLength": 1,
+              "enum": [
+                "auto",
+                "get",
+                "google_get",
+                "google_get_bypass"
+              ]
+            },
+            "max_retries": {
+              "type": "integer",
+              "not": {
+                "type": "null"
+              },
+              "minimum": 0,
+              "maximum": 3
+            },
+            "wait_for_selector": {
+              "type": [
+                "null",
+                "string"
+              ]
+            },
+            "wait_timeout_seconds": {
+              "type": "integer",
+              "not": {
+                "type": "null"
+              },
+              "exclusiveMinimum": 0
+            },
+            "block_images": {
+              "type": "boolean",
+              "not": {
+                "type": "null"
+              }
+            },
+            "block_images_and_css": {
+              "type": "boolean",
+              "not": {
+                "type": "null"
+              }
+            },
+            "wait_for_complete_page_load": {
+              "type": "boolean",
+              "not": {
+                "type": "null"
+              }
+            },
+            "headless": {
+              "type": "boolean",
+              "not": {
+                "type": "null"
+              }
+            },
+            "proxy": {
+              "type": "string",
+              "minLength": 1
+            },
+            "user_agent": {
+              "type": "string",
+              "minLength": 1
+            },
+            "window_size": {
+              "type": "array",
+              "items": {
+                "minLength": 2,
+                "maxLength": 2,
+                "type": "integer",
+                "exclusiveMinimum": 0
+              }
+            },
+            "lang": {
+              "type": "string",
+              "minLength": 1
+            }
+          },
+          "required": []
+        }
+      },
+      "required": []
+    }
+  },
+  "required": [
+    "channel"
+  ],
+  "anyOf": [
+    {
+      "required": [
+        "selectors"
+      ]
+    },
+    {
+      "required": [
+        "auto_source"
+      ]
+    }
+  ]
+}