automatic 14.12.2 → 26.08

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (172) hide show
  1. checksums.yaml +5 -5
  2. data/README.md +635 -83
  3. data/VERSION +1 -1
  4. data/automatic.gemspec +109 -248
  5. data/bin/automatic +20 -139
  6. data/config/feed2console.yml +10 -4
  7. data/config/feed2markdown.yml +41 -0
  8. data/doc/AI_TUTORIAL.md +518 -0
  9. data/doc/BASIC_DESIGN.md +516 -0
  10. data/doc/COPYING.LESSER +165 -0
  11. data/doc/DEPLOYMENT.md +824 -0
  12. data/doc/LICENSE.md +14 -0
  13. data/doc/PLUGINS.md +1875 -0
  14. data/doc/PLUGIN_DEVELOPMENT.md +86 -0
  15. data/doc/POLICY.md +857 -0
  16. data/doc/QUICKSTART.md +256 -0
  17. data/doc/RELEASING.md +381 -0
  18. data/doc/REQUIREMENTS.md +526 -0
  19. data/doc/VERSIONS +208 -0
  20. data/lib/automatic/cli.rb +248 -0
  21. data/lib/automatic/environment.rb +31 -5
  22. data/lib/automatic/feed_maker.rb +10 -9
  23. data/lib/automatic/feed_parser.rb +51 -35
  24. data/lib/automatic/http.rb +107 -0
  25. data/lib/automatic/log.rb +49 -18
  26. data/lib/automatic/opml.rb +3 -1
  27. data/lib/automatic/pipeline.rb +63 -32
  28. data/lib/automatic/recipe.rb +56 -17
  29. data/lib/automatic/version.rb +14 -1
  30. data/lib/automatic.rb +78 -20
  31. data/plugins/custom_feed/svn_log.rb +73 -32
  32. data/plugins/custom_feed/web.rb +348 -0
  33. data/plugins/filter/absolute_uri.rb +43 -27
  34. data/plugins/filter/accept.rb +38 -45
  35. data/plugins/filter/claude.rb +217 -0
  36. data/plugins/filter/clear.rb +12 -8
  37. data/plugins/filter/description_link.rb +49 -51
  38. data/plugins/filter/full_feed.rb +158 -52
  39. data/plugins/filter/gemini.rb +216 -0
  40. data/plugins/filter/github_feed.rb +38 -26
  41. data/plugins/filter/ignore.rb +33 -44
  42. data/plugins/filter/image.rb +36 -25
  43. data/plugins/filter/image_source.rb +58 -52
  44. data/plugins/filter/join.rb +107 -0
  45. data/plugins/filter/one.rb +19 -26
  46. data/plugins/filter/open_ai.rb +198 -0
  47. data/plugins/filter/rand.rb +16 -17
  48. data/plugins/filter/sakura_ai.rb +205 -0
  49. data/plugins/filter/sanitize.rb +29 -34
  50. data/plugins/filter/sort.rb +20 -27
  51. data/plugins/filter/tumblr_resize.rb +31 -23
  52. data/plugins/notify/ikachan.rb +86 -48
  53. data/plugins/provide/fluentd.rb +43 -24
  54. data/plugins/publish/amazon_s3.rb +73 -40
  55. data/plugins/publish/console.rb +19 -16
  56. data/plugins/publish/console_link.rb +20 -17
  57. data/plugins/publish/eject.rb +48 -26
  58. data/plugins/publish/fluentd.rb +50 -30
  59. data/plugins/publish/hatena_bookmark.rb +88 -71
  60. data/plugins/publish/instapaper.rb +69 -59
  61. data/plugins/publish/markdown.rb +278 -0
  62. data/plugins/publish/memcached.rb +35 -29
  63. data/plugins/store/database.rb +50 -48
  64. data/plugins/store/digest.rb +212 -0
  65. data/plugins/store/file.rb +99 -68
  66. data/plugins/store/full_text.rb +32 -25
  67. data/plugins/store/permalink.rb +18 -22
  68. data/plugins/subscription/feed.rb +34 -21
  69. data/plugins/subscription/link.rb +31 -32
  70. data/plugins/subscription/text.rb +32 -46
  71. data/plugins/subscription/tumblr.rb +55 -44
  72. data/plugins/subscription/xml.rb +40 -36
  73. metadata +108 -321
  74. data/Gemfile +0 -38
  75. data/Rakefile +0 -59
  76. data/doc/ChangeLog +0 -303
  77. data/doc/PLUGINS +0 -750
  78. data/doc/PLUGINS.ja +0 -753
  79. data/doc/README +0 -511
  80. data/doc/README.ja +0 -519
  81. data/plugins/filter/google_news.rb +0 -50
  82. data/plugins/publish/google_calendar.rb +0 -84
  83. data/plugins/publish/hipchat.rb +0 -46
  84. data/plugins/publish/pocket.rb +0 -45
  85. data/plugins/publish/twitter.rb +0 -58
  86. data/plugins/subscription/chan_toru.rb +0 -57
  87. data/plugins/subscription/g_guide.rb +0 -57
  88. data/plugins/subscription/pocket.rb +0 -51
  89. data/plugins/subscription/twitter.rb +0 -69
  90. data/plugins/subscription/twitter_search.rb +0 -50
  91. data/plugins/subscription/weather.rb +0 -33
  92. data/script/build +0 -84
  93. data/spec/fixtures/sampleFeeds.tsv +0 -1
  94. data/spec/fixtures/sampleFeeds2.tsv +0 -2
  95. data/spec/fixtures/sampleRecipe.yml +0 -24
  96. data/spec/lib/automatic/log_spec.rb +0 -32
  97. data/spec/lib/automatic/pipeline_spec.rb +0 -68
  98. data/spec/lib/automatic/recipe_spec.rb +0 -40
  99. data/spec/lib/automatic_spec.rb +0 -99
  100. data/spec/plugins/custom_feed/svn_log_spec.rb +0 -31
  101. data/spec/plugins/filter/absolute_uri_spec.rb +0 -61
  102. data/spec/plugins/filter/accept_spec.rb +0 -331
  103. data/spec/plugins/filter/clear_spec.rb +0 -49
  104. data/spec/plugins/filter/description_link_spec.rb +0 -138
  105. data/spec/plugins/filter/full_feed_spec.rb +0 -129
  106. data/spec/plugins/filter/github_feed_spec.rb +0 -55
  107. data/spec/plugins/filter/google_news_spec.rb +0 -69
  108. data/spec/plugins/filter/ignore_spec.rb +0 -328
  109. data/spec/plugins/filter/image_source_spec.rb +0 -89
  110. data/spec/plugins/filter/image_spec.rb +0 -65
  111. data/spec/plugins/filter/one_spec.rb +0 -71
  112. data/spec/plugins/filter/rand_spec.rb +0 -52
  113. data/spec/plugins/filter/sanitize_spec.rb +0 -153
  114. data/spec/plugins/filter/sort_spec.rb +0 -189
  115. data/spec/plugins/filter/tumblr_resize_spec.rb +0 -109
  116. data/spec/plugins/notify/ikachan_spec.rb +0 -58
  117. data/spec/plugins/provide/fluentd_spec.rb +0 -49
  118. data/spec/plugins/publish/amazon_s3_spec.rb +0 -40
  119. data/spec/plugins/publish/console_spec.rb +0 -30
  120. data/spec/plugins/publish/eject_spec.rb +0 -40
  121. data/spec/plugins/publish/fluentd_spec.rb +0 -40
  122. data/spec/plugins/publish/google_calendar_spec.rb +0 -83
  123. data/spec/plugins/publish/hatena_bookmark_spec.rb +0 -134
  124. data/spec/plugins/publish/hipchat_spec.rb +0 -69
  125. data/spec/plugins/publish/instapaper_spec.rb +0 -82
  126. data/spec/plugins/publish/memcached_spec.rb +0 -63
  127. data/spec/plugins/publish/pocket_spec.rb +0 -51
  128. data/spec/plugins/publish/twitter_spec.rb +0 -73
  129. data/spec/plugins/store/file_spec.rb +0 -58
  130. data/spec/plugins/store/full_text_spec.rb +0 -152
  131. data/spec/plugins/store/permalink_spec.rb +0 -206
  132. data/spec/plugins/subscription/chan_toru_spec.rb +0 -56
  133. data/spec/plugins/subscription/feed_spec.rb +0 -71
  134. data/spec/plugins/subscription/g_guide_spec.rb +0 -82
  135. data/spec/plugins/subscription/link_spec.rb +0 -72
  136. data/spec/plugins/subscription/pocket_spec.rb +0 -57
  137. data/spec/plugins/subscription/text_spec.rb +0 -84
  138. data/spec/plugins/subscription/tumblr_spec.rb +0 -74
  139. data/spec/plugins/subscription/twitter_search_spec.rb +0 -57
  140. data/spec/plugins/subscription/twitter_spec.rb +0 -73
  141. data/spec/plugins/subscription/weather_spec.rb +0 -44
  142. data/spec/plugins/subscription/xml_spec.rb +0 -84
  143. data/spec/spec_helper.rb +0 -106
  144. data/spec/user_dir/plugins/store/mock.rb +0 -16
  145. data/test/fixtures/sampleOPML.xml +0 -11
  146. data/test/integration/test_absoluteurl.yml +0 -25
  147. data/test/integration/test_activerecord.yml +0 -24
  148. data/test/integration/test_add_pocket.yml +0 -26
  149. data/test/integration/test_chan_toru.yml +0 -21
  150. data/test/integration/test_descriptionlink.yml +0 -21
  151. data/test/integration/test_fluentd.yml +0 -22
  152. data/test/integration/test_fulltext.yml +0 -30
  153. data/test/integration/test_google_news.yml +0 -21
  154. data/test/integration/test_googlealert.yml +0 -21
  155. data/test/integration/test_hatenabookmark.yml +0 -30
  156. data/test/integration/test_ignore.yml +0 -25
  157. data/test/integration/test_ignore2.yml +0 -22
  158. data/test/integration/test_image2local.yml +0 -33
  159. data/test/integration/test_instapaper.yml +0 -26
  160. data/test/integration/test_link2local.yml +0 -34
  161. data/test/integration/test_one.yml +0 -23
  162. data/test/integration/test_pocket.yml +0 -22
  163. data/test/integration/test_rand.yml +0 -21
  164. data/test/integration/test_sanitize.yml +0 -23
  165. data/test/integration/test_sort.yml +0 -36
  166. data/test/integration/test_svnlog.yml +0 -15
  167. data/test/integration/test_text2feed.yml +0 -36
  168. data/test/integration/test_tumblr2local.yml +0 -43
  169. data/test/integration/test_twitter_search.yml +0 -22
  170. data/test/integration/test_weather.yml +0 -19
  171. data/test/integration/test_xml2fluentd.yml +0 -21
  172. data/vendor/.gitkeep +0 -0
@@ -0,0 +1,516 @@
1
+ # Basic design: a plugin pipeline for unattended processing
2
+
3
+ ## 1. Purpose
4
+
5
+ This document says how Automatic Ruby is put together: what the pieces are, what
6
+ each is responsible for, which way they depend on each other, and how a value
7
+ travels through a run.
8
+
9
+ What the system is for belongs to [`REQUIREMENTS.md`](REQUIREMENTS.md). The
10
+ rules a change is held to belong to [`POLICY.md`](POLICY.md). The two public
11
+ contracts — the Recipe format and the plugin interface — are specified in
12
+ [`PLUGINS.md`](PLUGINS.md); this document explains the machinery that implements
13
+ them and does not restate them.
14
+
15
+ It stands on its own. Nothing in it is completed by a document kept in another
16
+ repository.
17
+
18
+ ## 2. Design policy
19
+
20
+ - **The framework is the small part.** It loads a Recipe, finds classes by name,
21
+ and calls them in order. It has no domain knowledge, and gaining some would be
22
+ a design error rather than a feature.
23
+ - **Plugins are the large part, and they are replaceable.** They are found on a
24
+ search path, not registered in a list, so that adding one touches no framework
25
+ file and removing one leaves nothing dangling.
26
+ - **One value connects everything.** Because every plugin takes and returns the
27
+ same shape, composition needs no adapter, no type negotiation and no schema.
28
+ - **The core is loadable without its plugins' dependencies.** `require
29
+ 'automatic'` pulls in the framework and nothing a plugin needs. Each plugin
30
+ requires its own libraries, at the top of its own file, so that an absent gem
31
+ is an error only for a Recipe that asked for that plugin.
32
+ - **The library never exits and never prints.** Exit status is decided by the
33
+ entry point; user-facing text is written by the entry point or logged.
34
+
35
+ ## 3. Composition
36
+
37
+ A typical run is a short, linear flow. Markdown is one publisher at the edge,
38
+ not a second pipeline representation:
39
+
40
+ ```text
41
+ source plugins -> shared pipeline -> optional filters / stores -> publishers
42
+ ```
43
+
44
+ ```text
45
+ bin/automatic process entry point; exit status only
46
+ |
47
+ v
48
+ lib/automatic/cli.rb option parsing, subcommands, error reporting
49
+ |
50
+ +---------------------------+
51
+ | |
52
+ v v
53
+ lib/automatic.rb lib/automatic/opml.rb diagnostic helpers
54
+ (run, directories) lib/automatic/feed_parser.rb
55
+ |
56
+ +-------------------+
57
+ | |
58
+ v v
59
+ lib/automatic/recipe.rb lib/automatic/pipeline.rb
60
+ | |
61
+ | v
62
+ | Automatic::Plugin::* plugins/<category>/<name>.rb
63
+ | | ~/.automatic/plugins/<category>/<name>.rb
64
+ v v
65
+ lib/automatic/log.rb lib/automatic/feed_maker.rb
66
+ | lib/automatic/http.rb
67
+ v
68
+ standard output
69
+ ```
70
+
71
+ Dependency points downward, and there is no edge back up:
72
+
73
+ - `bin/automatic` knows only `Automatic::CLI`.
74
+ - `Automatic::CLI` knows the framework. Nothing in the framework knows the CLI.
75
+ - `Automatic::Pipeline` knows how to find and call a plugin. It knows no plugin.
76
+ - A plugin knows `Automatic::Log`, `Automatic::FeedMaker`,
77
+ `Automatic::FeedParser`, `Automatic::Http` and its own libraries. It knows no
78
+ other plugin.
79
+ - `Automatic::Log` and `Automatic::FeedMaker` are leaves. They depend on nothing
80
+ in this repository.
81
+
82
+ `Automatic::Recipe` and `Automatic::Pipeline` do not know each other. Both are
83
+ driven by `Automatic.run`.
84
+
85
+ ## 4. What each part is for
86
+
87
+ ### 4.1 `bin/automatic`
88
+
89
+ The process entry point, and deliberately almost empty. It puts the
90
+ installation's `lib` on the load path, requires `automatic/cli`, and exits with
91
+ the status `Automatic::CLI.run(ARGV)` returns.
92
+
93
+ It contains no option definition, no subcommand and no policy. Everything a test
94
+ would want to exercise is therefore in a library file, reachable without
95
+ spawning a process.
96
+
97
+ ### 4.2 `lib/automatic/cli.rb` — `Automatic::CLI`
98
+
99
+ Everything that belongs to being a command:
100
+
101
+ - Builds the `OptionParser`: `-c/--config`, `-h/--help`, `-v/--version`.
102
+ - Holds the subcommand table (`scaffold`, `unscaffold`, `autodiscovery`,
103
+ `feedparser`, `inspect`, `opmlparser`, `log`) as a hash of name to callable.
104
+ - Implements `scaffold` and `unscaffold`, which are the only filesystem
105
+ operations the framework performs on its own behalf.
106
+ - Catches the framework's own exceptions, prints one line to standard error, and
107
+ returns the exit status.
108
+
109
+ `CLI.run` **returns** an `Integer` and never calls `exit`. That is what lets a
110
+ spec assert on a status and on captured output instead of a subprocess.
111
+
112
+ Exit status is decided in exactly one place:
113
+
114
+ | Status | Meaning |
115
+ | --- | --- |
116
+ | `0` | The Recipe ran, or the subcommand did its work, or help or version was printed |
117
+ | `1` | The run or the subcommand failed, or no work was requested |
118
+ | `2` | The command line was rejected by the option parser |
119
+
120
+ The libraries a subcommand needs are required inside that subcommand, not at the
121
+ top of the file, so running a Recipe loads neither `feedbag` nor the OPML
122
+ parser. The framework's own requires still apply: `require 'automatic'` pulls in
123
+ the Recipe loader, the pipeline, the log and the two feed adapters, and through
124
+ them Ruby's `rss`.
125
+
126
+ ### 4.3 `lib/automatic.rb` — `Automatic`
127
+
128
+ The module itself, holding the two directories the rest of the system resolves
129
+ paths against, and the one method that runs a job:
130
+
131
+ - `root_dir` — the installation root, set by the entry point.
132
+ - `user_dir` — `~/.automatic`, or an override when `AUTOMATIC_RUBY_ENV=test`,
133
+ which is how the specs point the loader at a fixture directory without writing
134
+ to a real home directory.
135
+ - `plugins_dir`, `config_dir`, `user_plugins_dir` — derived from those two.
136
+ - `run(recipe:, root_dir:, user_dir:)` — sets the directories and hands the
137
+ Recipe to `Pipeline.run`.
138
+
139
+ It `require`s the framework's own files and nothing else. It does not require a
140
+ plugin, and a plugin's dependency never appears here.
141
+
142
+ ### 4.4 `lib/automatic/environment.rb`
143
+
144
+ Bundler setup for a source checkout: if the repository's `Gemfile` is present,
145
+ set it up so `bin/automatic` resolves the locked gems.
146
+
147
+ It is a convenience for running from a checkout, not a requirement. When the gem
148
+ is installed normally there is no `Gemfile` to find and this file does nothing,
149
+ which is the correct behaviour: an installed library must not impose a bundle on
150
+ the program that requires it.
151
+
152
+ ### 4.5 `lib/automatic/recipe.rb` — `Automatic::Recipe`
153
+
154
+ Turns a Recipe file into an object the pipeline can iterate.
155
+
156
+ - Resolves the path: a bare name is looked for in `~/.automatic/config` first,
157
+ and anything not found there is treated as a path as given.
158
+ - Parses the YAML **safely** — the document may contain only the plain types a
159
+ Recipe needs, so a Recipe cannot name a Ruby class to instantiate. Aliases are
160
+ permitted, because they are a legitimate way to share a block of settings
161
+ between plugins. This is a second line of defence, not the trust boundary; see
162
+ [`REQUIREMENTS.md`](REQUIREMENTS.md) section 17.
163
+ - Wraps the result in `Hashie::Mash`, which is why a plugin entry answers to
164
+ `plugin.module` and `plugin.config`, and why an absent setting reads as `nil`
165
+ rather than raising.
166
+ - Applies `global.log.level` to `Automatic::Log`. This is the only `global` key
167
+ the framework reads.
168
+ - Exposes `each_plugin`, which yields the entries of `plugins` in order.
169
+ - Raises `Automatic::InvalidRecipeError` for a document that is not a mapping,
170
+ or that carries no usable `plugins` sequence.
171
+
172
+ It performs no validation of a plugin's own `config`. Whether a setting is
173
+ required, and what it must look like, is the plugin's business, because the
174
+ framework cannot know.
175
+
176
+ ### 4.6 `lib/automatic/pipeline.rb` — `Automatic::Pipeline`
177
+
178
+ The core, and the shortest file that matters.
179
+
180
+ `load_plugin(module_name)` resolves a class name to a file:
181
+
182
+ 1. `module_name.underscore` turns `SubscriptionFeed` into `subscription_feed`.
183
+ 2. The category directories are listed, `~/.automatic/plugins/*` **before**
184
+ `<root>/plugins/*`, so the user directory wins.
185
+ 3. For each, if the directory's own name is a prefix of the underscored module
186
+ name, the remainder is the file name: directory `subscription` and module
187
+ `subscription_feed` give `subscription/feed.rb`.
188
+ 4. The first existing file wins, and is registered with
189
+ `Automatic::Plugin.autoload`, so the file is read when the constant is first
190
+ used.
191
+ 5. Nothing matched raises `Automatic::NoPluginError` naming the module.
192
+
193
+ The category directory is therefore not a label: it is half of the lookup key.
194
+ This is what lets a plugin be added by dropping in a file, and a shipped plugin
195
+ be overridden by putting a file of the same name in the user directory.
196
+
197
+ `run(recipe)` is the pipeline:
198
+
199
+ ```ruby
200
+ pipeline = []
201
+ recipe.each_plugin do |plugin|
202
+ mod = plugin.module
203
+ load_plugin(mod)
204
+ klass = Automatic::Plugin.const_get(mod)
205
+ pipeline = klass.new(plugin.config, pipeline).run
206
+ end
207
+ ```
208
+
209
+ Each plugin's return value is the next plugin's input. There is no branching, no
210
+ inspection of the value between steps, and no exception handling: a plugin that
211
+ raises ends the run, for the reason given in `REQUIREMENTS.md` section 12.
212
+
213
+ ### 4.7 `lib/automatic/log.rb` — `Automatic::Log`
214
+
215
+ A `Logger` on standard output behind a level filter.
216
+
217
+ - `Log.level(name)` sets the threshold, from `global.log.level`.
218
+ - `Log.puts(level, message)` emits when `level` is at least the threshold.
219
+ - Levels are `info`, `warn`, `error`, `none`, compared by their position in that
220
+ list; `none` is the threshold that admits nothing.
221
+ - Both a `String` and a `Symbol` are accepted for a level, because plugins have
222
+ always passed both, and an unknown level is treated as `info` rather than
223
+ raising in the middle of an unattended run.
224
+
225
+ It is a module with state rather than an injected object. That is a consequence
226
+ of plugins calling `Automatic::Log` directly, which keeps a plugin's signature
227
+ to `(config, pipeline)`.
228
+
229
+ ### 4.8 `lib/automatic/feed_maker.rb`, `feed_parser.rb` and `http.rb`
230
+
231
+ The adapters between "some data" and the pipeline shape, and the one way in for
232
+ what is fetched.
233
+
234
+ - `FeedParser.get_url(url)` fetches a URL and parses it as a feed.
235
+ - `FeedParser.parse_html(html)` builds a feed whose items are the page's links,
236
+ which is how the link and Tumblr subscription plugins work.
237
+ - `FeedMaker.generate_feed(hash)` builds one item-like object from plain values.
238
+ - `FeedMaker.create_pipeline(items)` builds one feed object from a list of them.
239
+ Any plugin producing items from a non-feed source ends with this call.
240
+ - `FeedMaker.content_provide(url, data)` builds a one-item feed carrying an
241
+ arbitrary payload in `content_encoded`, which is the route by which the XML
242
+ subscription plugin feeds the Fluentd provide plugin.
243
+
244
+ The first two use Ruby's bundled `rss` library. That is the reason the pipeline
245
+ value has the shape it has.
246
+
247
+ - `Http.read(url)` fetches a URL and returns the body and `Http.open(url)`
248
+ yields the stream, for a caller — an HTML parser, which detects a page's
249
+ encoding for itself — that would rather not be handed a decoded string;
250
+ `Http.uri(url)` returns a validated URI and `Http.fetchable?(url)` answers
251
+ whether there is one.
252
+
253
+ `Automatic::Http` exists because the decisions a fetch implies — which schemes
254
+ are allowed, how long to wait, how many redirects to follow, what to send as a
255
+ User-Agent — were being made separately by every plugin that fetched, mostly by
256
+ omission. It is a helper of about twenty lines and not a client: a plugin that
257
+ wants something else calls Ruby directly. The scheme allowlist is the part that
258
+ earns it a file of its own, because a link in a pipeline item comes from a feed
259
+ and `URI.open` on such a string will read a local file as readily as an
260
+ article.
261
+
262
+ ### 4.9 `plugins/` — `Automatic::Plugin::*`
263
+
264
+ Every plugin is a class in `Automatic::Plugin`, constructed with `(config,
265
+ pipeline)` and answering `run`. The contract is specified in
266
+ [`PLUGINS.md`](PLUGINS.md) section 3; what matters to this document is how the
267
+ categories divide responsibility:
268
+
269
+ | Category | Directory | Receives | Returns | Role |
270
+ | --- | --- | --- | --- | --- |
271
+ | `Subscription` | `subscription/` | usually an empty pipeline | a pipeline | Acquire from outside |
272
+ | `CustomFeed` | `custom_feed/` | usually an empty pipeline | a pipeline | Build a feed from a non-feed source |
273
+ | `Filter` | `filter/` | a pipeline | a pipeline | Select, reorder or rewrite |
274
+ | `Store` | `store/` | a pipeline | a pipeline, usually reduced | Persist, and drop what was seen before |
275
+ | `Provide` | `provide/` | a pipeline | the same pipeline | Emit the payload elsewhere |
276
+ | `Notify` | `notify/` | a pipeline | the same pipeline | Send a notification |
277
+ | `Publish` | `publish/` | a pipeline | the same pipeline | Send the result out, or print it |
278
+
279
+ The categories are a convention with one mechanical consequence — the directory
280
+ name is part of the lookup key (section 4.6) — and no other. Nothing enforces
281
+ that a `Filter` does not reach the network.
282
+
283
+ **`Publish` is the boundary at which the pipeline meets a representation that is
284
+ not the pipeline's.** A publishing plugin reads the value described in section
285
+ 4.8, writes it out in whatever form its destination wants — a line on a
286
+ terminal, a request body, a record for a log collector, a document on disk — and
287
+ returns the value it was given, unchanged. The conversion is the plugin's whole
288
+ job and it happens in one direction, at one point:
289
+
290
+ ```text
291
+ RSS-shaped pipeline
292
+ |
293
+ v
294
+ Publish<Something> the plugin: serialize, then emit
295
+ |
296
+ v
297
+ the destination's own form a terminal line, a JSON record, a Markdown document
298
+ ```
299
+
300
+ `PublishMarkdown` is one of these and is architecturally unremarkable: it
301
+ renders each item as a Markdown section and writes the result to standard output
302
+ or to a file. What matters to this document is where it is *not*:
303
+
304
+ - **Not in the framework.** `Automatic::Pipeline` gains no knowledge of
305
+ Markdown, `Automatic::FeedMaker` gains no Markdown constructor, and no
306
+ framework file mentions the format. A reference to it in `lib/` would be the
307
+ same design error as a reference to any other plugin (section 3).
308
+ - **Not a second pipeline value.** Markdown is produced *from* the pipeline at
309
+ the moment the pipeline ends; it is never passed along one. The plugin returns
310
+ its input, so a plugin placed after it receives exactly what it would have
311
+ received without it, and Invariant 2 of [`POLICY.md`](POLICY.md) is untouched.
312
+ - **Not implicit.** Nothing appends it to a Recipe. `Pipeline.run` runs the
313
+ entries the Recipe lists, in order, and that is still the whole of its
314
+ behaviour.
315
+
316
+ Its specification — what it writes for each field, how HTML in a body is
317
+ reduced, where the output goes — is in [`PLUGINS.md`](PLUGINS.md) section 6.7,
318
+ because it is a plugin's specification and not a property of the design.
319
+
320
+ One shared piece sits inside `plugins/` rather than in `lib/`, because it is
321
+ plugin implementation and the framework does not use it:
322
+
323
+ - `plugins/store/database.rb` — the `Automatic::Plugin::Database` mixin: opens
324
+ the SQLite database named in the Recipe, creates the table from the including
325
+ class's `column_definition` when it is absent, and provides
326
+ `for_each_new_feed`, which yields only items whose key is not already stored.
327
+ `StorePermalink` and `StoreFullText` are this mixin plus a model and a column
328
+ list. `StoreDigest` takes the database part of it and decides for itself what
329
+ has been seen, because it is identified by a digest of an item's content
330
+ rather than by the link `for_each_new_feed` reads.
331
+ ### 4.10 `db/`, `config/`, `assets/`
332
+
333
+ Fallbacks inside the installation, used when the corresponding part of the user
334
+ directory is absent: `db/` for SQLite files, `config/` for the example Recipes
335
+ that `scaffold` copies out, `assets/` for data files a plugin needs.
336
+
337
+ ## 5. The flow of one run
338
+
339
+ ```text
340
+ automatic -c feed2console.yml
341
+ |
342
+ | CLI parses the command line, resolves the recipe path
343
+ v
344
+ Recipe.new(path)
345
+ | reads YAML safely, wraps in Hashie::Mash
346
+ | sets the log level from global.log.level
347
+ v
348
+ Automatic.run(recipe:, root_dir:)
349
+ | sets root_dir and user_dir
350
+ v
351
+ Pipeline.run(recipe)
352
+ |
353
+ | pipeline = []
354
+ |
355
+ | entry 1: module SubscriptionFeed
356
+ | load_plugin -> plugins/subscription/feed.rb
357
+ | SubscriptionFeed.new(config, []).run
358
+ | FeedParser.get_url(each configured feed)
359
+ | -> [feed]
360
+ |
361
+ | entry 2: module FilterIgnore
362
+ | load_plugin -> plugins/filter/ignore.rb
363
+ | FilterIgnore.new(config, [feed]).run
364
+ | drops items matching a keyword
365
+ | FeedMaker.create_pipeline(kept items)
366
+ | -> [feed']
367
+ |
368
+ | entry 3: module StorePermalink
369
+ | load_plugin -> plugins/store/permalink.rb
370
+ | StorePermalink.new(config, [feed']).run
371
+ | opens ~/.automatic/db/<db>, creates the table if absent
372
+ | for_each_new_feed: skips links already stored, inserts the rest
373
+ | -> [feed''] containing only what had not been seen
374
+ |
375
+ | entry 4: module PublishConsole
376
+ | PublishConsole.new(config, [feed'']).run
377
+ | prints each item
378
+ | -> [feed'']
379
+ v
380
+ the final value is discarded; CLI returns 0
381
+ ```
382
+
383
+ Three properties of that flow are the design:
384
+
385
+ - The pipeline **narrows**. Subscription plugins produce, filters and stores
386
+ reduce, publishers consume. A Recipe is normally read in that order.
387
+ - A store plugin is what makes a Recipe safe to run every five minutes. It is
388
+ the only thing standing between the operator and a duplicate.
389
+ - Nothing between the steps inspects the value. The framework never looks inside
390
+ a feed.
391
+
392
+ The last entry decides what the run leaves behind, and changing it changes
393
+ nothing else. The same four steps ending in `PublishMarkdown` produce a document
394
+ instead of terminal output:
395
+
396
+ ```text
397
+ acquire SubscriptionFeed, SubscriptionLink, CustomFeed*, ...
398
+ | many sources, one shape
399
+ v
400
+ filter FilterIgnore, FilterSanitize, FilterSort, ...
401
+ |
402
+ v
403
+ store StorePermalink: what has not been seen before
404
+ |
405
+ v
406
+ publish PublishMarkdown: serialize at the boundary
407
+ |
408
+ v
409
+ Markdown a file, or standard output
410
+ |
411
+ +--> read by a person
412
+ +--> searched with grep, processed with the usual tools
413
+ +--> committed, diffed, kept
414
+ +--> handed to a program, a language model or an agent
415
+ ```
416
+
417
+ Everything above the last box is unchanged from the previous diagram, which is
418
+ the point: the acquisition, the filtering and the de-duplication are the same
419
+ plugins over the same value, and the exit is the part the Recipe chooses.
420
+
421
+ ## 6. Settings
422
+
423
+ Two kinds, and they do not mix.
424
+
425
+ **Framework settings** come from the Recipe's `global` mapping, and there is one
426
+ that is read: `global.log.level`. `global.timezone` and `global.cache` appear in
427
+ the shipped examples and in Recipes in the wild, and nothing reads them. They
428
+ are kept — removing them would break nothing but would edit operators' files for
429
+ no gain — and they are documented as inert in
430
+ [`PLUGINS.md`](PLUGINS.md) section 2.4 so that no one adds a behaviour to them
431
+ by accident.
432
+
433
+ **Plugin settings** are the `config` mapping of a plugin entry, passed to the
434
+ constructor and read by nobody else. The framework does not validate, default,
435
+ merge or type-check them. A plugin that needs a value it was not given decides
436
+ what to do, and the common choice — treating an absent `retry` or `interval` as
437
+ zero — is why `nil.to_i` appears throughout the plugins.
438
+
439
+ There is no environment-variable configuration, with one exception:
440
+ `AUTOMATIC_RUBY_ENV=test` permits the user directory to be overridden, and
441
+ exists for the tests.
442
+
443
+ ## 7. Error handling
444
+
445
+ The layers handle errors differently, and on purpose.
446
+
447
+ **Plugins** own transient failure. A plugin that reaches the network wraps the
448
+ attempt, logs the failure, and retries `retry` times with `interval` seconds
449
+ between attempts. What a plugin must not do is swallow an error and return a
450
+ value that looks like success.
451
+
452
+ **The framework** owns nothing it cannot fix, so it catches nothing. Its own
453
+ failures are named:
454
+
455
+ | Exception | Raised when |
456
+ | --- | --- |
457
+ | `Automatic::NoRecipeError` | `Pipeline.run` was given no Recipe |
458
+ | `Automatic::NoPluginError` | No file was found for a module named in a Recipe |
459
+ | `Automatic::InvalidRecipeError` | The Recipe is not a mapping, or has no plugins |
460
+
461
+ All three derive from `Automatic::Error`, so a caller can rescue the framework's
462
+ failures without rescuing everything.
463
+
464
+ **The CLI** is the only place that turns an exception into a message and a
465
+ status. It reports the framework's errors as one line on standard error and
466
+ returns `1`. Anything else it lets propagate, because an unexpected exception is
467
+ a defect and its backtrace is wanted.
468
+
469
+ ## 8. Logging and output
470
+
471
+ - A library file never calls `puts`. It logs.
472
+ - `Automatic::CLI` writes to standard error for diagnostics and standard output
473
+ for requested output — help, version, and the results of the diagnostic
474
+ subcommands.
475
+ - Publishing plugins whose entire purpose is to print (`PublishConsole`,
476
+ `PublishConsoleLink`, `PublishMarkdown` with no `file` setting) write to an
477
+ output object held in an instance variable, defaulting to `$stdout`. That is
478
+ what lets their specs assert on what was printed by substituting a double.
479
+
480
+ The log and a publishing plugin's output therefore share standard output, which
481
+ has never mattered — a `pretty_inspect` dump and a log line interleave without
482
+ either becoming unreadable — and matters for a plugin whose output is a document
483
+ meant to be redirected into a file. The design does not resolve this by giving
484
+ the log a second destination: `Automatic::Log` writing to standard output is
485
+ long-standing behaviour that Recipes and `cron` entries are written around
486
+ (`REQUIREMENTS.md` section 14), and changing it for one plugin's benefit would
487
+ be the framework acquiring a plugin's problem. It is resolved where the choice
488
+ already exists — in the Recipe:
489
+
490
+ - `global.log.level: none` silences the log for that run, leaving standard
491
+ output carrying the document alone;
492
+ - or the plugin writes to a file it is given, and the log keeps standard output.
493
+
494
+ Both are stated in [`PLUGINS.md`](PLUGINS.md) section 6.7 and in
495
+ [`DEPLOYMENT.md`](DEPLOYMENT.md), which is where an operator looks.
496
+
497
+ ## 9. Testability
498
+
499
+ The design choices that exist for the tests:
500
+
501
+ - `CLI.run` returns a status instead of exiting, so command-line behaviour is a
502
+ unit test.
503
+ - `Automatic.user_dir=` accepts an override under `AUTOMATIC_RUBY_ENV=test`, so
504
+ the plugin loader can be pointed at `spec/user_dir` and the user-directory
505
+ precedence rule can be asserted. `spec/user_dir/plugins/store/mock.rb` exists
506
+ for exactly that.
507
+ - The plugin contract is a constructor and one method with no ambient input, so
508
+ a plugin spec is: build a pipeline, construct, `run`, assert on the result.
509
+ `AutomaticSpec.generate_pipeline` builds the pipeline.
510
+ - Each plugin requires its own libraries in its own file, so a plugin whose gem
511
+ is not installed is one spec that does not load, rather than a suite that does
512
+ not start.
513
+
514
+ The default suite covers the framework and the plugins that need neither the
515
+ network nor a credential. What that leaves out, and why, is in
516
+ [`PLUGINS.md`](PLUGINS.md) section 6 and in the README's testing section.
@@ -0,0 +1,165 @@
1
+ GNU LESSER GENERAL PUBLIC LICENSE
2
+ Version 3, 29 June 2007
3
+
4
+ Copyright (C) 2007 Free Software Foundation, Inc. <https://fsf.org/>
5
+ Everyone is permitted to copy and distribute verbatim copies
6
+ of this license document, but changing it is not allowed.
7
+
8
+
9
+ This version of the GNU Lesser General Public License incorporates
10
+ the terms and conditions of version 3 of the GNU General Public
11
+ License, supplemented by the additional permissions listed below.
12
+
13
+ 0. Additional Definitions.
14
+
15
+ As used herein, "this License" refers to version 3 of the GNU Lesser
16
+ General Public License, and the "GNU GPL" refers to version 3 of the GNU
17
+ General Public License.
18
+
19
+ "The Library" refers to a covered work governed by this License,
20
+ other than an Application or a Combined Work as defined below.
21
+
22
+ An "Application" is any work that makes use of an interface provided
23
+ by the Library, but which is not otherwise based on the Library.
24
+ Defining a subclass of a class defined by the Library is deemed a mode
25
+ of using an interface provided by the Library.
26
+
27
+ A "Combined Work" is a work produced by combining or linking an
28
+ Application with the Library. The particular version of the Library
29
+ with which the Combined Work was made is also called the "Linked
30
+ Version".
31
+
32
+ The "Minimal Corresponding Source" for a Combined Work means the
33
+ Corresponding Source for the Combined Work, excluding any source code
34
+ for portions of the Combined Work that, considered in isolation, are
35
+ based on the Application, and not on the Linked Version.
36
+
37
+ The "Corresponding Application Code" for a Combined Work means the
38
+ object code and/or source code for the Application, including any data
39
+ and utility programs needed for reproducing the Combined Work from the
40
+ Application, but excluding the System Libraries of the Combined Work.
41
+
42
+ 1. Exception to Section 3 of the GNU GPL.
43
+
44
+ You may convey a covered work under sections 3 and 4 of this License
45
+ without being bound by section 3 of the GNU GPL.
46
+
47
+ 2. Conveying Modified Versions.
48
+
49
+ If you modify a copy of the Library, and, in your modifications, a
50
+ facility refers to a function or data to be supplied by an Application
51
+ that uses the facility (other than as an argument passed when the
52
+ facility is invoked), then you may convey a copy of the modified
53
+ version:
54
+
55
+ a) under this License, provided that you make a good faith effort to
56
+ ensure that, in the event an Application does not supply the
57
+ function or data, the facility still operates, and performs
58
+ whatever part of its purpose remains meaningful, or
59
+
60
+ b) under the GNU GPL, with none of the additional permissions of
61
+ this License applicable to that copy.
62
+
63
+ 3. Object Code Incorporating Material from Library Header Files.
64
+
65
+ The object code form of an Application may incorporate material from
66
+ a header file that is part of the Library. You may convey such object
67
+ code under terms of your choice, provided that, if the incorporated
68
+ material is not limited to numerical parameters, data structure
69
+ layouts and accessors, or small macros, inline functions and templates
70
+ (ten or fewer lines in length), you do both of the following:
71
+
72
+ a) Give prominent notice with each copy of the object code that the
73
+ Library is used in it and that the Library and its use are
74
+ covered by this License.
75
+
76
+ b) Accompany the object code with a copy of the GNU GPL and this license
77
+ document.
78
+
79
+ 4. Combined Works.
80
+
81
+ You may convey a Combined Work under terms of your choice that,
82
+ taken together, effectively do not restrict modification of the
83
+ portions of the Library contained in the Combined Work and reverse
84
+ engineering for debugging such modifications, if you also do each of
85
+ the following:
86
+
87
+ a) Give prominent notice with each copy of the Combined Work that
88
+ the Library is used in it and that the Library and its use are
89
+ covered by this License.
90
+
91
+ b) Accompany the Combined Work with a copy of the GNU GPL and this license
92
+ document.
93
+
94
+ c) For a Combined Work that displays copyright notices during
95
+ execution, include the copyright notice for the Library among
96
+ these notices, as well as a reference directing the user to the
97
+ copies of the GNU GPL and this license document.
98
+
99
+ d) Do one of the following:
100
+
101
+ 0) Convey the Minimal Corresponding Source under the terms of this
102
+ License, and the Corresponding Application Code in a form
103
+ suitable for, and under terms that permit, the user to
104
+ recombine or relink the Application with a modified version of
105
+ the Linked Version to produce a modified Combined Work, in the
106
+ manner specified by section 6 of the GNU GPL for conveying
107
+ Corresponding Source.
108
+
109
+ 1) Use a suitable shared library mechanism for linking with the
110
+ Library. A suitable mechanism is one that (a) uses at run time
111
+ a copy of the Library already present on the user's computer
112
+ system, and (b) will operate properly with a modified version
113
+ of the Library that is interface-compatible with the Linked
114
+ Version.
115
+
116
+ e) Provide Installation Information, but only if you would otherwise
117
+ be required to provide such information under section 6 of the
118
+ GNU GPL, and only to the extent that such information is
119
+ necessary to install and execute a modified version of the
120
+ Combined Work produced by recombining or relinking the
121
+ Application with a modified version of the Linked Version. (If
122
+ you use option 4d0, the Installation Information must accompany
123
+ the Minimal Corresponding Source and Corresponding Application
124
+ Code. If you use option 4d1, you must provide the Installation
125
+ Information in the manner specified by section 6 of the GNU GPL
126
+ for conveying Corresponding Source.)
127
+
128
+ 5. Combined Libraries.
129
+
130
+ You may place library facilities that are a work based on the
131
+ Library side by side in a single library together with other library
132
+ facilities that are not Applications and are not covered by this
133
+ License, and convey such a combined library under terms of your
134
+ choice, if you do both of the following:
135
+
136
+ a) Accompany the combined library with a copy of the same work based
137
+ on the Library, uncombined with any other library facilities,
138
+ conveyed under the terms of this License.
139
+
140
+ b) Give prominent notice with the combined library that part of it
141
+ is a work based on the Library, and explaining where to find the
142
+ accompanying uncombined form of the same work.
143
+
144
+ 6. Revised Versions of the GNU Lesser General Public License.
145
+
146
+ The Free Software Foundation may publish revised and/or new versions
147
+ of the GNU Lesser General Public License from time to time. Such new
148
+ versions will be similar in spirit to the present version, but may
149
+ differ in detail to address new problems or concerns.
150
+
151
+ Each version is given a distinguishing version number. If the
152
+ Library as you received it specifies that a certain numbered version
153
+ of the GNU Lesser General Public License "or any later version"
154
+ applies to it, you have the option of following the terms and
155
+ conditions either of that published version or of any later version
156
+ published by the Free Software Foundation. If the Library as you
157
+ received it does not specify a version number of the GNU Lesser
158
+ General Public License, you may choose any version of the GNU Lesser
159
+ General Public License ever published by the Free Software Foundation.
160
+
161
+ If the Library as you received it specifies that a proxy can decide
162
+ whether future versions of the GNU Lesser General Public License shall
163
+ apply, that proxy's public statement of acceptance of any version is
164
+ permanent authorization for you to choose that version for the
165
+ Library.