inat-get 0.8.0.16 → 0.9.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of inat-get might be problematic. Click here for more details.

Files changed (192) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +278 -10
  3. data/bin/inat-get +4 -55
  4. data/lib/inat-get/app/application.rb +66 -0
  5. data/lib/inat-get/app/core/api.rb +65 -0
  6. data/lib/inat-get/app/core/console.rb +109 -0
  7. data/lib/inat-get/app/core/console_logger.rb +27 -0
  8. data/lib/inat-get/app/core/server.rb +145 -0
  9. data/lib/inat-get/app/core/task.rb +87 -0
  10. data/lib/inat-get/app/core/worker.rb +87 -0
  11. data/lib/inat-get/app/maintenance.rb +189 -0
  12. data/lib/inat-get/app/setup.rb +290 -0
  13. data/lib/inat-get/app.rb +5 -0
  14. data/lib/inat-get/data/dsl/conditions/and.rb +189 -0
  15. data/lib/inat-get/data/dsl/conditions/base.rb +203 -0
  16. data/lib/inat-get/data/dsl/conditions/not.rb +142 -0
  17. data/lib/inat-get/data/dsl/conditions/or.rb +261 -0
  18. data/lib/inat-get/data/dsl/conditions/query.rb +154 -0
  19. data/lib/inat-get/data/dsl/conditions/special/project_users.rb +18 -0
  20. data/lib/inat-get/data/dsl/conditions.rb +6 -0
  21. data/lib/inat-get/data/dsl/dataset.rb +196 -0
  22. data/lib/inat-get/data/dsl/dsl.rb +233 -0
  23. data/lib/inat-get/data/dsl/list.rb +261 -0
  24. data/lib/inat-get/data/helpers/base.rb +133 -0
  25. data/lib/inat-get/data/helpers/defs/accuracy.rb +33 -0
  26. data/lib/inat-get/data/helpers/defs/coordinate.rb +29 -0
  27. data/lib/inat-get/data/helpers/defs/datepart.rb +17 -0
  28. data/lib/inat-get/data/helpers/defs/has.rb +36 -0
  29. data/lib/inat-get/data/helpers/defs/iconic.rb +29 -0
  30. data/lib/inat-get/data/helpers/defs/identified.rb +19 -0
  31. data/lib/inat-get/data/helpers/defs/ids.rb +56 -0
  32. data/lib/inat-get/data/helpers/defs/licensed.rb +19 -0
  33. data/lib/inat-get/data/helpers/defs/licenses.rb +17 -0
  34. data/lib/inat-get/data/helpers/defs/location.rb +18 -0
  35. data/lib/inat-get/data/helpers/defs/models.rb +11 -0
  36. data/lib/inat-get/data/helpers/defs/period.rb +59 -0
  37. data/lib/inat-get/data/helpers/defs/place.rb +16 -0
  38. data/lib/inat-get/data/helpers/defs/project.rb +17 -0
  39. data/lib/inat-get/data/helpers/defs/range.rb +29 -0
  40. data/lib/inat-get/data/helpers/defs/rank.rb +63 -0
  41. data/lib/inat-get/data/helpers/defs/scalar.rb +16 -0
  42. data/lib/inat-get/data/helpers/defs/scalarcoord.rb +20 -0
  43. data/lib/inat-get/data/helpers/defs/scalarlocation.rb +14 -0
  44. data/lib/inat-get/data/helpers/defs/scalarmodel.rb +15 -0
  45. data/lib/inat-get/data/helpers/defs/set.rb +40 -0
  46. data/lib/inat-get/data/helpers/defs/taxon.rb +20 -0
  47. data/lib/inat-get/data/helpers/defs/verifiable.rb +19 -0
  48. data/lib/inat-get/data/helpers/defs.rb +44 -0
  49. data/lib/inat-get/data/helpers/identifications.rb +16 -0
  50. data/lib/inat-get/data/helpers/observations.rb +85 -0
  51. data/lib/inat-get/data/helpers/places.rb +25 -0
  52. data/lib/inat-get/data/helpers/projects.rb +28 -0
  53. data/lib/inat-get/data/helpers/taxa.rb +23 -0
  54. data/lib/inat-get/data/helpers/users.rb +24 -0
  55. data/lib/inat-get/data/managers/base.rb +144 -0
  56. data/lib/inat-get/data/managers/identifications.rb +60 -0
  57. data/lib/inat-get/data/managers/observations.rb +96 -0
  58. data/lib/inat-get/data/managers/places.rb +55 -0
  59. data/lib/inat-get/data/managers/projects.rb +45 -0
  60. data/lib/inat-get/data/managers/taxa.rb +43 -0
  61. data/lib/inat-get/data/managers/users.rb +45 -0
  62. data/lib/inat-get/data/models/annotation.rb +31 -0
  63. data/lib/inat-get/data/models/base.rb +55 -0
  64. data/lib/inat-get/data/models/fave.rb +30 -0
  65. data/lib/inat-get/data/models/identification.rb +23 -0
  66. data/lib/inat-get/data/models/observation.rb +54 -0
  67. data/lib/inat-get/data/models/photo.rb +15 -0
  68. data/lib/inat-get/data/models/place.rb +29 -0
  69. data/lib/inat-get/data/models/project.rb +99 -0
  70. data/lib/inat-get/data/models/projectadmin.rb +25 -0
  71. data/lib/inat-get/data/models/projectplace.rb +15 -0
  72. data/lib/inat-get/data/models/projectqualitygrade.rb +17 -0
  73. data/lib/inat-get/data/models/projecttaxon.rb +15 -0
  74. data/lib/inat-get/data/models/projectterm.rb +17 -0
  75. data/lib/inat-get/data/models/projectuser.rb +15 -0
  76. data/lib/inat-get/data/models/request.rb +22 -0
  77. data/lib/inat-get/data/models/sound.rb +13 -0
  78. data/lib/inat-get/data/models/sub.rb +10 -0
  79. data/lib/inat-get/data/models/tag.rb +24 -0
  80. data/lib/inat-get/data/models/taxon.rb +104 -0
  81. data/lib/inat-get/data/models/user.rb +27 -0
  82. data/lib/inat-get/data/parsers/annotation.rb +21 -0
  83. data/lib/inat-get/data/parsers/base.rb +50 -0
  84. data/lib/inat-get/data/parsers/defs/ancestry.rb +27 -0
  85. data/lib/inat-get/data/parsers/defs/assmodel.rb +29 -0
  86. data/lib/inat-get/data/parsers/defs/cached.rb +11 -0
  87. data/lib/inat-get/data/parsers/defs/children.rb +15 -0
  88. data/lib/inat-get/data/parsers/defs/copy.rb +25 -0
  89. data/lib/inat-get/data/parsers/defs/details.rb +55 -0
  90. data/lib/inat-get/data/parsers/defs/fromtaxon.rb +18 -0
  91. data/lib/inat-get/data/parsers/defs/json.rb +20 -0
  92. data/lib/inat-get/data/parsers/defs/links.rb +18 -0
  93. data/lib/inat-get/data/parsers/defs/location.rb +25 -0
  94. data/lib/inat-get/data/parsers/defs/model.rb +29 -0
  95. data/lib/inat-get/data/parsers/defs/observed.rb +17 -0
  96. data/lib/inat-get/data/parsers/defs/pk.rb +30 -0
  97. data/lib/inat-get/data/parsers/defs/prjdefault.rb +9 -0
  98. data/lib/inat-get/data/parsers/defs/prjmembers.rb +19 -0
  99. data/lib/inat-get/data/parsers/defs/prjrules.rb +92 -0
  100. data/lib/inat-get/data/parsers/defs/prjsearch.rb +60 -0
  101. data/lib/inat-get/data/parsers/defs/prjtype.rb +13 -0
  102. data/lib/inat-get/data/parsers/defs/strlocation.rb +23 -0
  103. data/lib/inat-get/data/parsers/defs/subprojects.rb +24 -0
  104. data/lib/inat-get/data/parsers/defs/time.rb +20 -0
  105. data/lib/inat-get/data/parsers/defs.rb +145 -0
  106. data/lib/inat-get/data/parsers/fave.rb +22 -0
  107. data/lib/inat-get/data/parsers/identification.rb +22 -0
  108. data/lib/inat-get/data/parsers/observation.rb +66 -0
  109. data/lib/inat-get/data/parsers/photo.rb +24 -0
  110. data/lib/inat-get/data/parsers/place.rb +32 -0
  111. data/lib/inat-get/data/parsers/project.rb +48 -0
  112. data/lib/inat-get/data/parsers/projectadmin.rb +18 -0
  113. data/lib/inat-get/data/parsers/sound.rb +17 -0
  114. data/lib/inat-get/data/parsers/tag.rb +16 -0
  115. data/lib/inat-get/data/parsers/taxon.rb +31 -0
  116. data/lib/inat-get/data/parsers/user.rb +27 -0
  117. data/lib/inat-get/data/types/iconic.rb +44 -0
  118. data/lib/inat-get/data/types/rank.rb +79 -0
  119. data/lib/inat-get/data/updaters/base.rb +398 -0
  120. data/lib/inat-get/data/updaters/identifications.rb +19 -0
  121. data/lib/inat-get/data/updaters/observations.rb +25 -0
  122. data/lib/inat-get/data/updaters/places.rb +14 -0
  123. data/lib/inat-get/data/updaters/projects.rb +14 -0
  124. data/lib/inat-get/data/updaters/taxa.rb +22 -0
  125. data/lib/inat-get/data/updaters/users.rb +26 -0
  126. data/lib/inat-get/info.rb +23 -0
  127. data/lib/inat-get/sys/context.rb +44 -0
  128. data/lib/inat-get/utils/json.rb +46 -0
  129. data/lib/inat-get/utils/simple_singular.rb +29 -0
  130. data/lib/inat-get.rb +3 -0
  131. metadata +337 -77
  132. data/.yardopts +0 -6
  133. data/Rakefile +0 -4
  134. data/docs/logo.png +0 -0
  135. data/inat-get.gemspec +0 -33
  136. data/lib/extra/enum.rb +0 -188
  137. data/lib/extra/period.rb +0 -267
  138. data/lib/extra/uuid.rb +0 -90
  139. data/lib/inat/app/application.rb +0 -51
  140. data/lib/inat/app/config/messagelevel.rb +0 -24
  141. data/lib/inat/app/config/shiftage.rb +0 -24
  142. data/lib/inat/app/config/updatemode.rb +0 -20
  143. data/lib/inat/app/config.rb +0 -300
  144. data/lib/inat/app/globals.rb +0 -91
  145. data/lib/inat/app/info.rb +0 -23
  146. data/lib/inat/app/logging.rb +0 -40
  147. data/lib/inat/app/preamble.rb +0 -27
  148. data/lib/inat/app/status.rb +0 -68
  149. data/lib/inat/app/task/context.rb +0 -53
  150. data/lib/inat/app/task/dsl.rb +0 -26
  151. data/lib/inat/app/task.rb +0 -75
  152. data/lib/inat/data/api.rb +0 -247
  153. data/lib/inat/data/db.rb +0 -92
  154. data/lib/inat/data/ddl.rb +0 -37
  155. data/lib/inat/data/entity/comment.rb +0 -33
  156. data/lib/inat/data/entity/flag.rb +0 -28
  157. data/lib/inat/data/entity/identification.rb +0 -50
  158. data/lib/inat/data/entity/observation.rb +0 -185
  159. data/lib/inat/data/entity/observationphoto.rb +0 -29
  160. data/lib/inat/data/entity/observationsound.rb +0 -25
  161. data/lib/inat/data/entity/photo.rb +0 -37
  162. data/lib/inat/data/entity/place.rb +0 -66
  163. data/lib/inat/data/entity/project.rb +0 -99
  164. data/lib/inat/data/entity/projectadmin.rb +0 -21
  165. data/lib/inat/data/entity/projectobservationrule.rb +0 -49
  166. data/lib/inat/data/entity/request.rb +0 -62
  167. data/lib/inat/data/entity/sound.rb +0 -32
  168. data/lib/inat/data/entity/taxon.rb +0 -102
  169. data/lib/inat/data/entity/user.rb +0 -70
  170. data/lib/inat/data/entity/vote.rb +0 -26
  171. data/lib/inat/data/entity.rb +0 -312
  172. data/lib/inat/data/enums/conservationstatus.rb +0 -30
  173. data/lib/inat/data/enums/geoprivacy.rb +0 -16
  174. data/lib/inat/data/enums/iconictaxa.rb +0 -23
  175. data/lib/inat/data/enums/identificationcategory.rb +0 -13
  176. data/lib/inat/data/enums/licensecode.rb +0 -20
  177. data/lib/inat/data/enums/projectadminrole.rb +0 -11
  178. data/lib/inat/data/enums/projecttype.rb +0 -39
  179. data/lib/inat/data/enums/qualitygrade.rb +0 -12
  180. data/lib/inat/data/enums/rank.rb +0 -60
  181. data/lib/inat/data/model.rb +0 -600
  182. data/lib/inat/data/query.rb +0 -1152
  183. data/lib/inat/data/sets/dataset.rb +0 -108
  184. data/lib/inat/data/sets/list.rb +0 -195
  185. data/lib/inat/data/sets/listers.rb +0 -21
  186. data/lib/inat/data/sets/wrappers.rb +0 -166
  187. data/lib/inat/data/types/extras.rb +0 -88
  188. data/lib/inat/data/types/location.rb +0 -98
  189. data/lib/inat/data/types/std.rb +0 -288
  190. data/lib/inat/report/report_dsl.rb +0 -210
  191. data/lib/inat/report/table.rb +0 -148
  192. data/lib/inat/utils/deep.rb +0 -36
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: d8f300ef3ec85bd4a168f2d4cbda6cc8ed922bb080c3e7de57685e6d753e58ae
4
- data.tar.gz: 9e7cb3ddacd711e9d7b79c273e2f0811f698bb3596ed7421893335d2c525252e
3
+ metadata.gz: dbe428cb94175725066a61cdb1022523cba2a0e429613458ec532e4301432c8b
4
+ data.tar.gz: 82ce8aa99fe0dfe48da6e8c0d0fc7193a4b6a90e794815939862aca0105fc44e
5
5
  SHA512:
6
- metadata.gz: 190b1940be2c402d2b5ec8092618276f362b9d8eab8a732aa1dd439bc396240189fbb587dd3d7398fc67183c8d4d2c070ef791137bce967c264176ec0c914c3f
7
- data.tar.gz: 8356abb7580c8c43d8dd603eaa6a7e2e7a4b9d0bb72320104cdf699f170cbb7d1fb0938e1a7db350cb5515c9e74dba6237ddfa059f1fd705a1eb411f73dd6c2c
6
+ metadata.gz: 56e50506345683d97dbb93d97b9aa6c2c45b3d162feb1114959a24fb799500ed2a104762f438408e8148a86288cba4e99b0bd77557b9d358e8068a58ed370c8b
7
+ data.tar.gz: 37151bf35d79d932ec90bb737bac4f2dbefbae596dc3af368a29c7a791a3c4b383f03031f5911c6555208e5e53322e0acf7fa90c0f41bdbc917e196310e80458
data/README.md CHANGED
@@ -1,16 +1,284 @@
1
- <img src="docs/logo.png" align="right" style="float:right;width:57%;">
1
+ # 🌿 inat-get
2
2
 
3
- # INat::Get
3
+ [![GitHub License](https://img.shields.io/github/license/inat-get/inat-get)](LICENSE)
4
+ [![Gem Version](https://badge.fury.io/rb/inat-get.svg?icon=si%3Arubygems&d=1)](https://badge.fury.io/rb/inat-get)
5
+ [![Ruby](https://github.com/inat-get/inat-get/actions/workflows/ruby.yml/badge.svg)](https://github.com/inat-get/inat-get/actions/workflows/ruby.yml)
6
+ ![Coverage](coverage-badge.svg)
4
7
 
5
- A toolset for fetching and processing data from **[iNaturalist.org][inat]**.
8
+ ## What is this and why?
6
9
 
7
- The author of this software is not associated or affiliated with the iNaturalist.
8
- Only the [public API][api] is used in accordance with [the conditions][cond].
10
+ `inat-get` is a utility for fetching and analyzing data from **[iNaturalist](https://www.inaturalist.org/)**.
9
11
 
10
- [inat]: https://www.inaturalist.org/
11
- [api]: https://api.inaturalist.org/v1/docs/
12
- [cond]: https://api.inaturalist.org/v1/docs/#terms-of-use
12
+ The basic approach is to form queries as *declaratively* as possible and get reports,
13
+ without giving up advanced capabilities. This leads us to the concept of **{file:DSL.md DSL — Domain Specific Language}** —
14
+ and user scripts using it. It is assumed that the user will need a *minimal*
15
+ familiarity with Ruby syntax, but if desired, they can use the full power of the language.
13
16
 
14
- ## Current status
17
+ The second crucial aspect is **caching**, designed to minimize duplication of
18
+ requested data without compromising freshness. For caching, a local database is used,
19
+ which potentially can be from a fairly wide range of supported DBMS: SQLite, PostgreSQL, MySQL
20
+ and others.
15
21
 
16
- Early alpha version.
22
+ *However, it should be noted that version 0.9.0 was tested only on SQLite3, full testing
23
+ with various DBMS is planned for version 0.9.4...*
24
+
25
+ The third key point is **parallel execution** of multiple user scripts: first,
26
+ almost all computers are multi-core now, and second, while one is waiting for a response from the API, others can
27
+ calmly do processing. At the same time, the API requests themselves go through a single synchronous point to strictly
28
+ comply with the restrictions imposed by the iNaturalist API itself, and not get banned for incorrect use.
29
+
30
+ ### What is not and will not be
31
+
32
+ + *Authorization and data submission.* This is a tool for analysis, not for automating any actions
33
+ with iNaturalist.
34
+
35
+ + *Neural network integration.* Strict analytics and neural networks should not be mixed. Of course, integration
36
+ "from the other side" is possible, i.e., fetching data and then using them with neural networks.
37
+
38
+ + *Native versions for other systems.* Currently the script works only on Linux (though possibly under
39
+ other unix-like systems, not checked). Native cross-platform support is not planned, but containerization
40
+ is planned (though not soon), which will allow expanding usage.
41
+
42
+ ## Current status and plans
43
+
44
+ + **v0.8.x**
45
+
46
+ Outdated versions. Due to poorly thought-out architecture, it works very slowly and caches very poorly.
47
+ Can be used, but shouldn't be.
48
+
49
+ + **[v0.9.0](https://github.com/inat-get/inat-get/milestone/4)** *(current development)*
50
+
51
+ Architectural problems are solved, but because of them the project had to be rewritten almost from scratch, and a lot remains
52
+ to be done.
53
+
54
+ *This is more of a beta than an alpha, but a very early beta.*
55
+
56
+ The main unfinished piece is the absence of a report formatting system. Below in the [Examples](#examples) section you can
57
+ see how this is worked around... Besides, not all query types are implemented and caching is unfinished
58
+ in terms of updating *old* data.
59
+
60
+ + **[v0.9.2](https://github.com/inat-get/inat-get/milestone/5)**
61
+
62
+ Planning to add ERB support for reports. It's not a huge step forward compared to
63
+ direct file writing, but it will add some convenience.
64
+
65
+ + **[v0.9.4](https://github.com/inat-get/inat-get/milestone/6)**
66
+
67
+ Planning to conduct extensive load testing on different DBMS (at least SQLite and PostgreSQL),
68
+ followed by corrections and optimizations.
69
+
70
+ That is, version 0.9.4 is planned as a strictly corrective and optimizing release without adding new functionality.
71
+
72
+ + **[v0.9.6](https://github.com/inat-get/inat-get/milestone/7)**
73
+
74
+ Here the completion of work with queries is planned — implementation of missing ones, finalization of the caching system, etc.
75
+
76
+ + **[v0.9.8](https://github.com/inat-get/inat-get/milestone/8)**
77
+
78
+ This version is planned for the development and debugging of a general reporting system. It's not yet entirely clear what exactly
79
+ it will be, the task is in `research` status...
80
+
81
+ + **[v1.0](https://github.com/inat-get/inat-get/milestone/9)** <u>(release)</u>
82
+
83
+ Basically, everything above, refactored and polished. Of the serious changes visible to the user,
84
+ only *containerization* and, through it, cross-platform support is planned.
85
+
86
+ + **[v1.2](https://github.com/inat-get/inat-get/milestone/10)**
87
+
88
+ Plans are not yet fully formed and will change. In versions after 1.0, advanced
89
+ capabilities are planned to be added, the exact composition of which will become clearer in the process of real usage.
90
+
91
+ + **v2.0**
92
+
93
+ Very uncertain future, by that time the iNaturalist API will likely have changed, and completely
94
+ new needs will have appeared...
95
+
96
+ ## Installation and usage
97
+
98
+ ### Installation
99
+
100
+ The project is packaged as a Ruby Gem, so it is installed accordingly:
101
+ ```shell
102
+ $ gem install inat-get
103
+ ```
104
+
105
+ However, if you want to use the current version "from sources", you can simply clone the repository
106
+ and run `bundle install`:
107
+ ```shell
108
+ $ git clone https://github.com/inat-get/inat-get.git
109
+ $ cd inat-get
110
+ $ bundle install
111
+ ```
112
+
113
+ And then run via `bundle exec`:
114
+ ```shell
115
+ $ bundle exec inat-get [options] ‹task› [‹task› ...]
116
+ ```
117
+
118
+ ### Command line parameters
119
+
120
+ <pre><b>$</b> bundle exec inat-get --help
121
+ 🌿 <b>iNatGet v0.9.0:</b> iNaturalist API query builder and analytics tool
122
+ License: <b>GNU GPLv3+</b> (https://github.com/inat-get/inat-get/blob/main/LICENSE)
123
+ Author: <b>Ivan Shikhalev</b> (https://github.com/shikhalev)
124
+ Homepage: <b>https://github.com/inat-get/inat-get</b>
125
+
126
+ <b>Usage:</b> inat-get [options] ‹task› [‹task› ...]
127
+
128
+ <b>Info Options:</b>
129
+ -h, --help Show this help and exit.
130
+ --version Show version and exit.
131
+ -i, --info Show information about DB status and API connection.
132
+ Then exit.
133
+ --show-config Show current configuration and exit.
134
+
135
+ <b>Main Options:</b>
136
+ -c, --config FILE Use this file as config (must be YAML)
137
+ [default: ~/.config/inat-get.yml].
138
+ -l, --log-level LEVEL Log level (fatal, error, warn, info or debug)
139
+ [default: warn].
140
+ --debug Set log level to debug.
141
+ -o, --offline Offline mode: no updates, use local database only.
142
+ -O, --online Online mode [default], use this flag to cancel
143
+ 'offline: true' in config.
144
+
145
+ <b>DB Maintenance:</b>
146
+ -C, --db-check Check DB version and exit.
147
+ -U, --db-update Migrate to latest DB version and exit.
148
+ -M, --db-migrate VER Migrate to DB version VER and exit.
149
+ --db-create Create database (error if exists).
150
+ --db-reset Drop (if exists) and recreate database. All fetched
151
+ data will be lost.
152
+
153
+ <b>File Arguments:</b>
154
+ ‹task› [‹task› ...] One or more names of task files or list files with '@'
155
+ prefix (one task file per line). If task name has not
156
+ extension try to read '‹task›' than '‹task›.inat' than
157
+ '‹task›.rb'.
158
+ </pre>
159
+
160
+ ### DSL
161
+
162
+ So, in scripts you can (and should) use specially prepared DSL methods and objects
163
+ responsible for data.
164
+
165
+ First of all, these are **models**, documentation for which can be found at
166
+ <https://inat-get.github.io/inat-get/INatGet/Data/Model.html>. In general, these are simply data
167
+ objects linked to each other. The key ones are probably:
168
+ [`Observation`](https://inat-get.github.io/inat-get/INatGet/Data/Model/Observation.html),
169
+ [`Taxon`](https://inat-get.github.io/inat-get/INatGet/Data/Model/Taxon.html),
170
+ [`Project`](https://inat-get.github.io/inat-get/INatGet/Data/Model/Project.html),
171
+ [`Place`](https://inat-get.github.io/inat-get/INatGet/Data/Model/Place.html)
172
+ and [`User`](https://inat-get.github.io/inat-get/INatGet/Data/Model/User.html).
173
+
174
+ The main work is built on the arithmetic of **datasets** and **lists**. Datasets are data
175
+ selections, manipulations with which happen without real access to the API and DB until
176
+ it becomes necessary. Lists are datasets split by some key field.
177
+ Documentation is at <https://inat-get.github.io/inat-get/INatGet/Data/DSL/Dataset.html>
178
+ for `Dataset` and <https://inat-get.github.io/inat-get/INatGet/Data/DSL/List.html> for `List`.
179
+ Both classes implement the `Enumerable` module, each in its own way...
180
+
181
+ At the beginning of each script we make observation selections (usually) via the `select_observations` method.
182
+ In general, `select_*` and `get_*` methods for different object types are described in the DSL module documentation —
183
+ <https://inat-get.github.io/inat-get/INatGet/Data/DSL.html>. In user scripts this
184
+ module is initially included in the context, so its methods are available directly.
185
+
186
+ *Unfortunately, the fields of selections are not yet documented.* Will be soon.
187
+
188
+ Below are simple examples of user scripts. It is recommended to pay attention
189
+ to "arithmetic" operations: `+`, `-`, `*` and `%`. The last one performs splitting and turns
190
+ a dataset into a list.
191
+
192
+ ## Examples
193
+
194
+ ### Simple report for a user — [user_stat.rb](share/inat-get/demo/01_user_stat.rb)
195
+
196
+ ```ruby
197
+ # Let's make a simple report on taxa observed by the user since the beginning of the year.
198
+ # The report will be output to the current directory with the name user_stat.md (Markdown format)
199
+
200
+ year = today.year
201
+
202
+ user = get_user 'shikhalev' # Here specify the user ID or login, I specified my own
203
+
204
+ # Get observations
205
+ observations = select_observations user: user, observed: time_range(year: year), quality_grade: 'research'
206
+
207
+ by_taxon = observations % :taxon
208
+
209
+ File::open "#{ name }.md", 'w' do |file|
210
+ file.puts '## Report for user ' + user.login + (user.name ? " (#{ user.name })" : '')
211
+ file.puts ''
212
+ by_taxon.each do |ds|
213
+ # Here ds.key is a Taxon object
214
+ file.puts "+ #{ ds.key.common_name } *(#{ ds.key.name })* — #{ ds.count } obs."
215
+ end
216
+ file.puts ''
217
+ file.puts "Total **#{ observations.count }** observations"
218
+ end
219
+ ```
220
+
221
+ ### Demonstration of list subtraction — [underbound.rb](share/inat-get/demo/02_underfound.rb)
222
+
223
+ ```ruby
224
+ # And here we implement the following: for a certain area, find a list of taxa
225
+ # that the given user has not observed (but others have).
226
+
227
+ user = get_user 'shikhalev'
228
+ place = get_place 'artinskiy-gorodskoy-okrug-osm-2023-sv-ru'
229
+
230
+ all_observations = select_observations place: place, quality_grade: 'research', rank: (.. Rank.complex)
231
+ full_list = all_observations % :taxon
232
+
233
+ user_observations = select_observations place: place, quality_grade: "research", rank: (.. Rank.complex), user: user
234
+ user_list = user_observations % :taxon
235
+
236
+ others_list = full_list - user_list
237
+ others_list.sort! { |ds| -ds.count }
238
+
239
+ File::open "#{ name }.md", 'w' do |file|
240
+ file.puts '## Not found by you'
241
+ file.puts ''
242
+ others_list.each do |ds|
243
+ file.puts "+ #{ ds.key.common_name } *(#{ ds.key.name })* — #{ ds.count } obs."
244
+ end
245
+ file.puts ''
246
+ file.puts "Total **#{ others_list.count }** taxa."
247
+ end
248
+ ```
249
+
250
+ ### List filtering and date range — [newcomers.rb](share/inat-get/demo/03_newcomers.rb)
251
+
252
+ ```ruby
253
+ # Newcomers of the previous month. As simple as possible: those who made an observation during
254
+ # the previous month, and registered in it as well. Naturally, within some
255
+ # project, so as not to pull too much.
256
+
257
+ project = get_project 'bioraznoobrazie-rayonov-sverdlovskoy-oblasti'
258
+
259
+ month = today.month - 1
260
+ year = if month == 0
261
+ month = 12
262
+ today.year - 1
263
+ else
264
+ today.year
265
+ end
266
+
267
+ period = time_range year: year, month: month
268
+ observations = select_observations project: project, created: period
269
+
270
+ list = observations % :user
271
+ list.filter! { |ds| period === ds.key.created }
272
+ list.sort! { |ds| ds.key.created }
273
+
274
+ File.open "#{ name }.md", 'w' do |file|
275
+ file.puts "\#\# Newcomers of project «#{ project.title }»"
276
+ file.puts "*#{ period.begin.to_date } — #{ period.end.to_date - 1 }*"
277
+ file.puts ''
278
+ list.each do |ds|
279
+ file.puts "+ #{ ds.key.login } (#{ ds.key.created.to_date }) — #{ ds.count } obs."
280
+ end
281
+ file.puts ''
282
+ file.puts "Total #{ list.count } users"
283
+ end
284
+ ```
data/bin/inat-get CHANGED
@@ -1,59 +1,8 @@
1
- #!/usr/bin/ruby -w
1
+ #!/usr/bin/ruby --jit
2
2
 
3
- require 'date'
4
- require 'pp'
3
+ require_relative '../lib/inat-get'
4
+ require_relative '../lib/inat-get/app'
5
5
 
6
- # require 'extra/enum'
6
+ app = INatGet::App::Main::instance
7
7
 
8
- require 'inat/app/application'
9
- require 'inat/app/globals'
10
- require 'inat/data/entity/observation'
11
- require 'inat/data/entity/request'
12
- require 'inat/data/db'
13
- require 'inat/data/query'
14
-
15
- app = INat::Application.new
16
8
  app.run
17
- # PP::pp Globals.config, $>, 64
18
-
19
- # puts
20
- # puts DDL.DDL
21
-
22
- # PP::pp DB::instance, $>, 64
23
-
24
- # data = API::query(:observations, user_login: 'shikhalev', month: 11)
25
-
26
- # PP::pp data.map { |d| Observation::parse(d) }.size, $>, 64
27
-
28
- # https://www.inaturalist.org/projects/174222
29
- # https://www.inaturalist.org/places/193592
30
- # https://www.inaturalist.org/places/193881 (Ачитский район)
31
- # https://www.inaturalist.org/places/194104 (Богдановичский район)
32
- # https://www.inaturalist.org/places/194414 (Качканарский район)
33
- # https://www.inaturalist.org/places/194023 (Берёзовский район)
34
- # https://api.inaturalist.org/v1/observations?project_id=176067&updated_since=2023-10-31T19:36:29+05:00&per_page=200&order_by=id&order=asc&locale=ru&preferred_place_id=11829
35
-
36
- # W, [2023-11-02T01:54:41.818021 #8118] WARN -- ‹main›: Some Taxon IDs were not fetched: 48460, 1, 47120, 245097, 47119, 47118, 120474, 342614, 319384, 67599, 495875, 153683, 153680, 367182, 48893, 1252003, 60920!
37
- # W, [2023-11-02T03:49:58.509031 #8118] WARN -- ‹main›: Some Taxon IDs were not fetched: 1, 47118, 47119, 47120, 48460, 48893, 60920, 67599, 120474, 153680, 153683, 245097, 319384, 342614, 367182, 495875, 1252003!
38
-
39
- # query = Query::new project_id: 180212
40
- # PP::pp query.observations.size, $>, 64
41
-
42
- # require 'inat/data/types/apiquery'
43
-
44
- # pp Time::new.to_date.to_s
45
-
46
- # a = ApiQuery::new ololo: 1, lalala: UUID.generate
47
-
48
- # PP::pp a, $>, 64
49
- # puts a.to_query
50
-
51
- # Request::create ''
52
-
53
- # pp URI::parse('https://api.inaturalist.org/v1/observations?project_id=176067&updated_since=2023-10-31T19:36:29+05:00&per_page=200&order_by=id&order=asc&locale=ru&preferred_place_id=11829')
54
-
55
- # d = Date::parse '2023-11-01'
56
- # t = d.to_time
57
- # i = t.to_i
58
-
59
- # pp [ d, t, i ]
@@ -0,0 +1,66 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'logger'
4
+ require 'singleton'
5
+
6
+ require_relative '../info'
7
+ require_relative 'core/console'
8
+ require_relative 'core/api'
9
+ require_relative 'core/task'
10
+ require_relative 'core/worker'
11
+
12
+ module INatGet::App; end
13
+
14
+ class INatGet::App::Main
15
+
16
+ include Singleton
17
+
18
+ def initialize
19
+ @config = INatGet::App::Setup::config!
20
+ if @config[:tasks].nil?
21
+ warn "❌ No tasks specified!"
22
+ exit Errno::ECHILD::Errno
23
+ end
24
+ end
25
+
26
+ def run
27
+ console_socket = @config.dig :socket, :console
28
+ api_socket = @config.dig :socket, :api
29
+ check_sockets! api_socket, console_socket
30
+
31
+ INatGet::App::Maintenance::db_check @config, true
32
+
33
+ console = INatGet::App::Server::Console::create console_socket
34
+ api = nil
35
+ api = INatGet::App::Server::API::create api_socket, console: console if !@config[:offline]
36
+
37
+ tasks = @config[:tasks].map { |path| INatGet::App::Task::new path, @config, console: console, api: api }
38
+ Process::warmup
39
+ INatGet::App::Worker::enqueue @config, *tasks, console: console, api: api
40
+ console.quit
41
+ api.quit if api
42
+ end
43
+
44
+ private
45
+
46
+ def check_sockets! socket, socket2
47
+ if File.exist?(socket)
48
+ if socket_alive?(socket)
49
+ warn "❌ API Socket already exists!"
50
+ exit Errno::EEXIST::Errno
51
+ else
52
+ File.delete socket
53
+ end
54
+ end
55
+
56
+ if File.exist?(socket2)
57
+ File.delete socket2
58
+ end
59
+ end
60
+
61
+ def socket_alive? socket
62
+ return false unless File.socket?(socket)
63
+ INatGet::App::Server::used? socket
64
+ end
65
+
66
+ end
@@ -0,0 +1,65 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'faraday'
4
+ require 'faraday/retry'
5
+ require 'is-duration'
6
+
7
+ require_relative 'server'
8
+ require_relative 'console_logger'
9
+
10
+ class INatGet::App::Server::API < INatGet::App::Server
11
+
12
+ def initialize socket_path, **params
13
+ @console = params.delete :console
14
+ @logger = ::INatGet::App::ConsoleLogger::new @console, progname: 'API'
15
+ super(socket_path, **params)
16
+ @config = INatGet::App::Setup::config
17
+ @delay = IS::Duration::parse @config.dig(:api, :delay)
18
+ end
19
+
20
+ private
21
+
22
+ def get query, **opts
23
+ endpoint = @config.dig(:api, :root) + query[:endpoint].to_s
24
+ timepoint = Time::now
25
+ if @last_request
26
+ delta = timepoint - @last_request
27
+ sleep (@delay - delta) if delta < @delay
28
+ end
29
+ @last_request = timepoint
30
+ response = faraday.get(endpoint) do |rq|
31
+ rq.params[:per_page] = @config.dig(:api, :pager)
32
+ rq.params.compact!
33
+ rq.params.merge! query[:query]
34
+ rq.headers["User-Agent"] = "iNatGet v#{INatGet::Info::VERSION} (#{ INatGet::Info::VERSION_ALIAS })"
35
+ end
36
+ if response.success?
37
+ begin
38
+ data = JSON.parse response.body, symbolize_names: true
39
+ return data.freeze
40
+ rescue => e
41
+ @logger.error "Error while parsing: #{e.message}"
42
+ return { status: :error, error: e.message }.freeze
43
+ end
44
+ else
45
+ @logger.error "Error in response: #{response.status}"
46
+ return { status: :error, error: response.status }.freeze
47
+ end
48
+ end
49
+
50
+ def faraday
51
+ # tmp_logger = ::Logger::new 'common.log', level: :info
52
+ @faraday ||= Faraday::new do |f|
53
+ f.request :retry,
54
+ max: @config.dig(:api, :retry, :max),
55
+ interval: IS::Duration::parse(@config.dig(:api, :retry, :interval)),
56
+ interval_randomness: @config.dig(:api, :retry, :randomness),
57
+ backoff_factor: @config.dig(:api, :retry, :backoff),
58
+ exceptions: [Faraday::TimeoutError, Faraday::ConnectionFailed, Faraday::SSLError, Faraday::ClientError]
59
+ f.request :url_encoded
60
+ # f.response :logger, tmp_logger, bodies: true, headers: true
61
+ f.adapter Faraday::default_adapter
62
+ end
63
+ end
64
+
65
+ end
@@ -0,0 +1,109 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'logger'
4
+
5
+ require 'is-term/statustable'
6
+ require 'is-term/formats'
7
+ require 'is-term/functions'
8
+
9
+ require_relative 'server'
10
+
11
+ class INatGet::App::Server::Console < INatGet::App::Server
12
+
13
+ def initialize socket_path, **params
14
+ super(socket_path, **params)
15
+ @table = IS::Term::StatusTable::instance
16
+ @table.configure do
17
+ column :icon, func: lambda { |row| row[:_active] ? "\e[1m[ ]" : "[✔]" }
18
+ separator ' '
19
+ column :_sender_pid, id: true, align: :right
20
+ separator ' '
21
+ column :name
22
+ separator ' '
23
+ column :status, align: :center
24
+ separator ' '
25
+ column :current, align: :right, summary: :current
26
+ separator ' of '
27
+ column :total, align: :left, summary: :total
28
+ separator ' '
29
+ column :percent, align: :right, func: :percent, format: '%d%%', summary: :percent
30
+ separator ' '
31
+ column :estimated, func: :estimated, align: :right, format: :duration, summary: :elapsed
32
+ separator ' '
33
+ column :speed, func: :speed, format: '%.2f r/s', align: :right, summary: :speed
34
+ separator ' '
35
+ column :message, summary: :value
36
+
37
+ summary true, message: ''
38
+ end
39
+ if @config.dig(:logs, :file, :enable)
40
+ filename = @config.dig(:logs, :file, :enable)
41
+ @sys_logger = Logger::new filename, 'daily', 7, level: @config.dig(:logs, :file, :sys)
42
+ @api_logger = Logger::new filename, 'daily', 7, level: @config.dig(:logs, :file, :api)
43
+ @wrk_logger = Logger::new filename, 'daily', 7, level: @config.dig(:logs, :file, :wrk)
44
+ end
45
+ @log_levels = {
46
+ sys: Logger::Severity::coerce(@config.dig(:logs, :screen, :sys)),
47
+ api: Logger::Severity::coerce(@config.dig(:logs, :screen, :api)),
48
+ wrk: Logger::Severity::coerce(@config.dig(:logs, :screen, :wrk))
49
+ }
50
+ end
51
+
52
+ def register **opts
53
+ @table.append(**opts)
54
+ end
55
+
56
+ def update **opts
57
+ @table.update(**opts)
58
+ end
59
+
60
+ def log severity, message, progname, **opts
61
+ key = prog_key progname
62
+ pid = opts[:_sender_pid]
63
+ console_log severity, message, progname, key, pid
64
+ file_log severity, message, progname, key if @config.dig(:logs, :file, :enable)
65
+ true
66
+ end
67
+
68
+ private
69
+
70
+ def prog_key progname
71
+ if progname.nil? || progname == ''
72
+ :sys
73
+ elsif progname == 'SYS' || progname == 'API'
74
+ progname.downcase.to_sym
75
+ else
76
+ :wrk
77
+ end
78
+ end
79
+
80
+ def file_log severity, message, progname, prog_key
81
+ case prog_key
82
+ when :api
83
+ @api_logger.log severity, message, progname
84
+ when :wrk
85
+ @wrk_logger.log severity, message, progname
86
+ else
87
+ @sys_logger.log severity, message, progname
88
+ end
89
+ end
90
+
91
+ SEV_ICON = {
92
+ Logger::Severity::DEBUG => '📓',
93
+ Logger::Severity::INFO => '📢',
94
+ Logger::Severity::WARN => '🔔',
95
+ Logger::Severity::ERROR => '🚨',
96
+ Logger::Severity::FATAL => '❌'
97
+ }
98
+
99
+ def console_log severity, message, progname, prog_key, pid
100
+ if severity >= @log_levels[prog_key]
101
+ if prog_key == :wrk
102
+ @table.update _sender_pid: pid, message: "#{ SEV_ICON[severity] } #{ message }"
103
+ else
104
+ @table.summary message: "#{ SEV_ICON[severity] } [#{ progname }] #{ message }"
105
+ end
106
+ end
107
+ end
108
+
109
+ end
@@ -0,0 +1,27 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'logger'
4
+
5
+ require_relative '../../info'
6
+
7
+ module INatGet::App; end
8
+
9
+ class INatGet::App::ConsoleLogger < Logger
10
+
11
+ def initialize console, **options
12
+ @console = console
13
+ super(nil, **options)
14
+ end
15
+
16
+ def add severity, msg = nil, progname = nil
17
+ if block_given?
18
+ msg = yield
19
+ elsif msg.nil?
20
+ msg = progname
21
+ progname = nil
22
+ end
23
+ return if msg.nil?
24
+ @console.log severity, msg, (progname || self.progname)
25
+ end
26
+
27
+ end