inat-get 0.8.0.16 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of inat-get might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/README.md +278 -10
- data/bin/inat-get +4 -55
- data/lib/inat-get/app/application.rb +66 -0
- data/lib/inat-get/app/core/api.rb +65 -0
- data/lib/inat-get/app/core/console.rb +109 -0
- data/lib/inat-get/app/core/console_logger.rb +27 -0
- data/lib/inat-get/app/core/server.rb +145 -0
- data/lib/inat-get/app/core/task.rb +87 -0
- data/lib/inat-get/app/core/worker.rb +87 -0
- data/lib/inat-get/app/maintenance.rb +189 -0
- data/lib/inat-get/app/setup.rb +290 -0
- data/lib/inat-get/app.rb +5 -0
- data/lib/inat-get/data/dsl/conditions/and.rb +189 -0
- data/lib/inat-get/data/dsl/conditions/base.rb +203 -0
- data/lib/inat-get/data/dsl/conditions/not.rb +142 -0
- data/lib/inat-get/data/dsl/conditions/or.rb +261 -0
- data/lib/inat-get/data/dsl/conditions/query.rb +154 -0
- data/lib/inat-get/data/dsl/conditions/special/project_users.rb +18 -0
- data/lib/inat-get/data/dsl/conditions.rb +6 -0
- data/lib/inat-get/data/dsl/dataset.rb +196 -0
- data/lib/inat-get/data/dsl/dsl.rb +233 -0
- data/lib/inat-get/data/dsl/list.rb +261 -0
- data/lib/inat-get/data/helpers/base.rb +133 -0
- data/lib/inat-get/data/helpers/defs/accuracy.rb +33 -0
- data/lib/inat-get/data/helpers/defs/coordinate.rb +29 -0
- data/lib/inat-get/data/helpers/defs/datepart.rb +17 -0
- data/lib/inat-get/data/helpers/defs/has.rb +36 -0
- data/lib/inat-get/data/helpers/defs/iconic.rb +29 -0
- data/lib/inat-get/data/helpers/defs/identified.rb +19 -0
- data/lib/inat-get/data/helpers/defs/ids.rb +56 -0
- data/lib/inat-get/data/helpers/defs/licensed.rb +19 -0
- data/lib/inat-get/data/helpers/defs/licenses.rb +17 -0
- data/lib/inat-get/data/helpers/defs/location.rb +18 -0
- data/lib/inat-get/data/helpers/defs/models.rb +11 -0
- data/lib/inat-get/data/helpers/defs/period.rb +59 -0
- data/lib/inat-get/data/helpers/defs/place.rb +16 -0
- data/lib/inat-get/data/helpers/defs/project.rb +17 -0
- data/lib/inat-get/data/helpers/defs/range.rb +29 -0
- data/lib/inat-get/data/helpers/defs/rank.rb +63 -0
- data/lib/inat-get/data/helpers/defs/scalar.rb +16 -0
- data/lib/inat-get/data/helpers/defs/scalarcoord.rb +20 -0
- data/lib/inat-get/data/helpers/defs/scalarlocation.rb +14 -0
- data/lib/inat-get/data/helpers/defs/scalarmodel.rb +15 -0
- data/lib/inat-get/data/helpers/defs/set.rb +40 -0
- data/lib/inat-get/data/helpers/defs/taxon.rb +20 -0
- data/lib/inat-get/data/helpers/defs/verifiable.rb +19 -0
- data/lib/inat-get/data/helpers/defs.rb +44 -0
- data/lib/inat-get/data/helpers/identifications.rb +16 -0
- data/lib/inat-get/data/helpers/observations.rb +85 -0
- data/lib/inat-get/data/helpers/places.rb +25 -0
- data/lib/inat-get/data/helpers/projects.rb +28 -0
- data/lib/inat-get/data/helpers/taxa.rb +23 -0
- data/lib/inat-get/data/helpers/users.rb +24 -0
- data/lib/inat-get/data/managers/base.rb +144 -0
- data/lib/inat-get/data/managers/identifications.rb +60 -0
- data/lib/inat-get/data/managers/observations.rb +96 -0
- data/lib/inat-get/data/managers/places.rb +55 -0
- data/lib/inat-get/data/managers/projects.rb +45 -0
- data/lib/inat-get/data/managers/taxa.rb +43 -0
- data/lib/inat-get/data/managers/users.rb +45 -0
- data/lib/inat-get/data/models/annotation.rb +31 -0
- data/lib/inat-get/data/models/base.rb +55 -0
- data/lib/inat-get/data/models/fave.rb +30 -0
- data/lib/inat-get/data/models/identification.rb +23 -0
- data/lib/inat-get/data/models/observation.rb +54 -0
- data/lib/inat-get/data/models/photo.rb +15 -0
- data/lib/inat-get/data/models/place.rb +29 -0
- data/lib/inat-get/data/models/project.rb +99 -0
- data/lib/inat-get/data/models/projectadmin.rb +25 -0
- data/lib/inat-get/data/models/projectplace.rb +15 -0
- data/lib/inat-get/data/models/projectqualitygrade.rb +17 -0
- data/lib/inat-get/data/models/projecttaxon.rb +15 -0
- data/lib/inat-get/data/models/projectterm.rb +17 -0
- data/lib/inat-get/data/models/projectuser.rb +15 -0
- data/lib/inat-get/data/models/request.rb +22 -0
- data/lib/inat-get/data/models/sound.rb +13 -0
- data/lib/inat-get/data/models/sub.rb +10 -0
- data/lib/inat-get/data/models/tag.rb +24 -0
- data/lib/inat-get/data/models/taxon.rb +104 -0
- data/lib/inat-get/data/models/user.rb +27 -0
- data/lib/inat-get/data/parsers/annotation.rb +21 -0
- data/lib/inat-get/data/parsers/base.rb +50 -0
- data/lib/inat-get/data/parsers/defs/ancestry.rb +27 -0
- data/lib/inat-get/data/parsers/defs/assmodel.rb +29 -0
- data/lib/inat-get/data/parsers/defs/cached.rb +11 -0
- data/lib/inat-get/data/parsers/defs/children.rb +15 -0
- data/lib/inat-get/data/parsers/defs/copy.rb +25 -0
- data/lib/inat-get/data/parsers/defs/details.rb +55 -0
- data/lib/inat-get/data/parsers/defs/fromtaxon.rb +18 -0
- data/lib/inat-get/data/parsers/defs/json.rb +20 -0
- data/lib/inat-get/data/parsers/defs/links.rb +18 -0
- data/lib/inat-get/data/parsers/defs/location.rb +25 -0
- data/lib/inat-get/data/parsers/defs/model.rb +29 -0
- data/lib/inat-get/data/parsers/defs/observed.rb +17 -0
- data/lib/inat-get/data/parsers/defs/pk.rb +30 -0
- data/lib/inat-get/data/parsers/defs/prjdefault.rb +9 -0
- data/lib/inat-get/data/parsers/defs/prjmembers.rb +19 -0
- data/lib/inat-get/data/parsers/defs/prjrules.rb +92 -0
- data/lib/inat-get/data/parsers/defs/prjsearch.rb +60 -0
- data/lib/inat-get/data/parsers/defs/prjtype.rb +13 -0
- data/lib/inat-get/data/parsers/defs/strlocation.rb +23 -0
- data/lib/inat-get/data/parsers/defs/subprojects.rb +24 -0
- data/lib/inat-get/data/parsers/defs/time.rb +20 -0
- data/lib/inat-get/data/parsers/defs.rb +145 -0
- data/lib/inat-get/data/parsers/fave.rb +22 -0
- data/lib/inat-get/data/parsers/identification.rb +22 -0
- data/lib/inat-get/data/parsers/observation.rb +66 -0
- data/lib/inat-get/data/parsers/photo.rb +24 -0
- data/lib/inat-get/data/parsers/place.rb +32 -0
- data/lib/inat-get/data/parsers/project.rb +48 -0
- data/lib/inat-get/data/parsers/projectadmin.rb +18 -0
- data/lib/inat-get/data/parsers/sound.rb +17 -0
- data/lib/inat-get/data/parsers/tag.rb +16 -0
- data/lib/inat-get/data/parsers/taxon.rb +31 -0
- data/lib/inat-get/data/parsers/user.rb +27 -0
- data/lib/inat-get/data/types/iconic.rb +44 -0
- data/lib/inat-get/data/types/rank.rb +79 -0
- data/lib/inat-get/data/updaters/base.rb +398 -0
- data/lib/inat-get/data/updaters/identifications.rb +19 -0
- data/lib/inat-get/data/updaters/observations.rb +25 -0
- data/lib/inat-get/data/updaters/places.rb +14 -0
- data/lib/inat-get/data/updaters/projects.rb +14 -0
- data/lib/inat-get/data/updaters/taxa.rb +22 -0
- data/lib/inat-get/data/updaters/users.rb +26 -0
- data/lib/inat-get/info.rb +23 -0
- data/lib/inat-get/sys/context.rb +44 -0
- data/lib/inat-get/utils/json.rb +46 -0
- data/lib/inat-get/utils/simple_singular.rb +29 -0
- data/lib/inat-get.rb +3 -0
- metadata +337 -77
- data/.yardopts +0 -6
- data/Rakefile +0 -4
- data/docs/logo.png +0 -0
- data/inat-get.gemspec +0 -33
- data/lib/extra/enum.rb +0 -188
- data/lib/extra/period.rb +0 -267
- data/lib/extra/uuid.rb +0 -90
- data/lib/inat/app/application.rb +0 -51
- data/lib/inat/app/config/messagelevel.rb +0 -24
- data/lib/inat/app/config/shiftage.rb +0 -24
- data/lib/inat/app/config/updatemode.rb +0 -20
- data/lib/inat/app/config.rb +0 -300
- data/lib/inat/app/globals.rb +0 -91
- data/lib/inat/app/info.rb +0 -23
- data/lib/inat/app/logging.rb +0 -40
- data/lib/inat/app/preamble.rb +0 -27
- data/lib/inat/app/status.rb +0 -68
- data/lib/inat/app/task/context.rb +0 -53
- data/lib/inat/app/task/dsl.rb +0 -26
- data/lib/inat/app/task.rb +0 -75
- data/lib/inat/data/api.rb +0 -247
- data/lib/inat/data/db.rb +0 -92
- data/lib/inat/data/ddl.rb +0 -37
- data/lib/inat/data/entity/comment.rb +0 -33
- data/lib/inat/data/entity/flag.rb +0 -28
- data/lib/inat/data/entity/identification.rb +0 -50
- data/lib/inat/data/entity/observation.rb +0 -185
- data/lib/inat/data/entity/observationphoto.rb +0 -29
- data/lib/inat/data/entity/observationsound.rb +0 -25
- data/lib/inat/data/entity/photo.rb +0 -37
- data/lib/inat/data/entity/place.rb +0 -66
- data/lib/inat/data/entity/project.rb +0 -99
- data/lib/inat/data/entity/projectadmin.rb +0 -21
- data/lib/inat/data/entity/projectobservationrule.rb +0 -49
- data/lib/inat/data/entity/request.rb +0 -62
- data/lib/inat/data/entity/sound.rb +0 -32
- data/lib/inat/data/entity/taxon.rb +0 -102
- data/lib/inat/data/entity/user.rb +0 -70
- data/lib/inat/data/entity/vote.rb +0 -26
- data/lib/inat/data/entity.rb +0 -312
- data/lib/inat/data/enums/conservationstatus.rb +0 -30
- data/lib/inat/data/enums/geoprivacy.rb +0 -16
- data/lib/inat/data/enums/iconictaxa.rb +0 -23
- data/lib/inat/data/enums/identificationcategory.rb +0 -13
- data/lib/inat/data/enums/licensecode.rb +0 -20
- data/lib/inat/data/enums/projectadminrole.rb +0 -11
- data/lib/inat/data/enums/projecttype.rb +0 -39
- data/lib/inat/data/enums/qualitygrade.rb +0 -12
- data/lib/inat/data/enums/rank.rb +0 -60
- data/lib/inat/data/model.rb +0 -600
- data/lib/inat/data/query.rb +0 -1152
- data/lib/inat/data/sets/dataset.rb +0 -108
- data/lib/inat/data/sets/list.rb +0 -195
- data/lib/inat/data/sets/listers.rb +0 -21
- data/lib/inat/data/sets/wrappers.rb +0 -166
- data/lib/inat/data/types/extras.rb +0 -88
- data/lib/inat/data/types/location.rb +0 -98
- data/lib/inat/data/types/std.rb +0 -288
- data/lib/inat/report/report_dsl.rb +0 -210
- data/lib/inat/report/table.rb +0 -148
- data/lib/inat/utils/deep.rb +0 -36
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: dbe428cb94175725066a61cdb1022523cba2a0e429613458ec532e4301432c8b
|
|
4
|
+
data.tar.gz: 82ce8aa99fe0dfe48da6e8c0d0fc7193a4b6a90e794815939862aca0105fc44e
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 56e50506345683d97dbb93d97b9aa6c2c45b3d162feb1114959a24fb799500ed2a104762f438408e8148a86288cba4e99b0bd77557b9d358e8068a58ed370c8b
|
|
7
|
+
data.tar.gz: 37151bf35d79d932ec90bb737bac4f2dbefbae596dc3af368a29c7a791a3c4b383f03031f5911c6555208e5e53322e0acf7fa90c0f41bdbc917e196310e80458
|
data/README.md
CHANGED
|
@@ -1,16 +1,284 @@
|
|
|
1
|
-
|
|
1
|
+
# 🌿 inat-get
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
[](LICENSE)
|
|
4
|
+
[](https://badge.fury.io/rb/inat-get)
|
|
5
|
+
[](https://github.com/inat-get/inat-get/actions/workflows/ruby.yml)
|
|
6
|
+

|
|
4
7
|
|
|
5
|
-
|
|
8
|
+
## What is this and why?
|
|
6
9
|
|
|
7
|
-
|
|
8
|
-
Only the [public API][api] is used in accordance with [the conditions][cond].
|
|
10
|
+
`inat-get` is a utility for fetching and analyzing data from **[iNaturalist](https://www.inaturalist.org/)**.
|
|
9
11
|
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
12
|
+
The basic approach is to form queries as *declaratively* as possible and get reports,
|
|
13
|
+
without giving up advanced capabilities. This leads us to the concept of **{file:DSL.md DSL — Domain Specific Language}** —
|
|
14
|
+
and user scripts using it. It is assumed that the user will need a *minimal*
|
|
15
|
+
familiarity with Ruby syntax, but if desired, they can use the full power of the language.
|
|
13
16
|
|
|
14
|
-
|
|
17
|
+
The second crucial aspect is **caching**, designed to minimize duplication of
|
|
18
|
+
requested data without compromising freshness. For caching, a local database is used,
|
|
19
|
+
which potentially can be from a fairly wide range of supported DBMS: SQLite, PostgreSQL, MySQL
|
|
20
|
+
and others.
|
|
15
21
|
|
|
16
|
-
|
|
22
|
+
*However, it should be noted that version 0.9.0 was tested only on SQLite3, full testing
|
|
23
|
+
with various DBMS is planned for version 0.9.4...*
|
|
24
|
+
|
|
25
|
+
The third key point is **parallel execution** of multiple user scripts: first,
|
|
26
|
+
almost all computers are multi-core now, and second, while one is waiting for a response from the API, others can
|
|
27
|
+
calmly do processing. At the same time, the API requests themselves go through a single synchronous point to strictly
|
|
28
|
+
comply with the restrictions imposed by the iNaturalist API itself, and not get banned for incorrect use.
|
|
29
|
+
|
|
30
|
+
### What is not and will not be
|
|
31
|
+
|
|
32
|
+
+ *Authorization and data submission.* This is a tool for analysis, not for automating any actions
|
|
33
|
+
with iNaturalist.
|
|
34
|
+
|
|
35
|
+
+ *Neural network integration.* Strict analytics and neural networks should not be mixed. Of course, integration
|
|
36
|
+
"from the other side" is possible, i.e., fetching data and then using them with neural networks.
|
|
37
|
+
|
|
38
|
+
+ *Native versions for other systems.* Currently the script works only on Linux (though possibly under
|
|
39
|
+
other unix-like systems, not checked). Native cross-platform support is not planned, but containerization
|
|
40
|
+
is planned (though not soon), which will allow expanding usage.
|
|
41
|
+
|
|
42
|
+
## Current status and plans
|
|
43
|
+
|
|
44
|
+
+ **v0.8.x**
|
|
45
|
+
|
|
46
|
+
Outdated versions. Due to poorly thought-out architecture, it works very slowly and caches very poorly.
|
|
47
|
+
Can be used, but shouldn't be.
|
|
48
|
+
|
|
49
|
+
+ **[v0.9.0](https://github.com/inat-get/inat-get/milestone/4)** *(current development)*
|
|
50
|
+
|
|
51
|
+
Architectural problems are solved, but because of them the project had to be rewritten almost from scratch, and a lot remains
|
|
52
|
+
to be done.
|
|
53
|
+
|
|
54
|
+
*This is more of a beta than an alpha, but a very early beta.*
|
|
55
|
+
|
|
56
|
+
The main unfinished piece is the absence of a report formatting system. Below in the [Examples](#examples) section you can
|
|
57
|
+
see how this is worked around... Besides, not all query types are implemented and caching is unfinished
|
|
58
|
+
in terms of updating *old* data.
|
|
59
|
+
|
|
60
|
+
+ **[v0.9.2](https://github.com/inat-get/inat-get/milestone/5)**
|
|
61
|
+
|
|
62
|
+
Planning to add ERB support for reports. It's not a huge step forward compared to
|
|
63
|
+
direct file writing, but it will add some convenience.
|
|
64
|
+
|
|
65
|
+
+ **[v0.9.4](https://github.com/inat-get/inat-get/milestone/6)**
|
|
66
|
+
|
|
67
|
+
Planning to conduct extensive load testing on different DBMS (at least SQLite and PostgreSQL),
|
|
68
|
+
followed by corrections and optimizations.
|
|
69
|
+
|
|
70
|
+
That is, version 0.9.4 is planned as a strictly corrective and optimizing release without adding new functionality.
|
|
71
|
+
|
|
72
|
+
+ **[v0.9.6](https://github.com/inat-get/inat-get/milestone/7)**
|
|
73
|
+
|
|
74
|
+
Here the completion of work with queries is planned — implementation of missing ones, finalization of the caching system, etc.
|
|
75
|
+
|
|
76
|
+
+ **[v0.9.8](https://github.com/inat-get/inat-get/milestone/8)**
|
|
77
|
+
|
|
78
|
+
This version is planned for the development and debugging of a general reporting system. It's not yet entirely clear what exactly
|
|
79
|
+
it will be, the task is in `research` status...
|
|
80
|
+
|
|
81
|
+
+ **[v1.0](https://github.com/inat-get/inat-get/milestone/9)** <u>(release)</u>
|
|
82
|
+
|
|
83
|
+
Basically, everything above, refactored and polished. Of the serious changes visible to the user,
|
|
84
|
+
only *containerization* and, through it, cross-platform support is planned.
|
|
85
|
+
|
|
86
|
+
+ **[v1.2](https://github.com/inat-get/inat-get/milestone/10)**
|
|
87
|
+
|
|
88
|
+
Plans are not yet fully formed and will change. In versions after 1.0, advanced
|
|
89
|
+
capabilities are planned to be added, the exact composition of which will become clearer in the process of real usage.
|
|
90
|
+
|
|
91
|
+
+ **v2.0**
|
|
92
|
+
|
|
93
|
+
Very uncertain future, by that time the iNaturalist API will likely have changed, and completely
|
|
94
|
+
new needs will have appeared...
|
|
95
|
+
|
|
96
|
+
## Installation and usage
|
|
97
|
+
|
|
98
|
+
### Installation
|
|
99
|
+
|
|
100
|
+
The project is packaged as a Ruby Gem, so it is installed accordingly:
|
|
101
|
+
```shell
|
|
102
|
+
$ gem install inat-get
|
|
103
|
+
```
|
|
104
|
+
|
|
105
|
+
However, if you want to use the current version "from sources", you can simply clone the repository
|
|
106
|
+
and run `bundle install`:
|
|
107
|
+
```shell
|
|
108
|
+
$ git clone https://github.com/inat-get/inat-get.git
|
|
109
|
+
$ cd inat-get
|
|
110
|
+
$ bundle install
|
|
111
|
+
```
|
|
112
|
+
|
|
113
|
+
And then run via `bundle exec`:
|
|
114
|
+
```shell
|
|
115
|
+
$ bundle exec inat-get [options] ‹task› [‹task› ...]
|
|
116
|
+
```
|
|
117
|
+
|
|
118
|
+
### Command line parameters
|
|
119
|
+
|
|
120
|
+
<pre><b>$</b> bundle exec inat-get --help
|
|
121
|
+
🌿 <b>iNatGet v0.9.0:</b> iNaturalist API query builder and analytics tool
|
|
122
|
+
License: <b>GNU GPLv3+</b> (https://github.com/inat-get/inat-get/blob/main/LICENSE)
|
|
123
|
+
Author: <b>Ivan Shikhalev</b> (https://github.com/shikhalev)
|
|
124
|
+
Homepage: <b>https://github.com/inat-get/inat-get</b>
|
|
125
|
+
|
|
126
|
+
<b>Usage:</b> inat-get [options] ‹task› [‹task› ...]
|
|
127
|
+
|
|
128
|
+
<b>Info Options:</b>
|
|
129
|
+
-h, --help Show this help and exit.
|
|
130
|
+
--version Show version and exit.
|
|
131
|
+
-i, --info Show information about DB status and API connection.
|
|
132
|
+
Then exit.
|
|
133
|
+
--show-config Show current configuration and exit.
|
|
134
|
+
|
|
135
|
+
<b>Main Options:</b>
|
|
136
|
+
-c, --config FILE Use this file as config (must be YAML)
|
|
137
|
+
[default: ~/.config/inat-get.yml].
|
|
138
|
+
-l, --log-level LEVEL Log level (fatal, error, warn, info or debug)
|
|
139
|
+
[default: warn].
|
|
140
|
+
--debug Set log level to debug.
|
|
141
|
+
-o, --offline Offline mode: no updates, use local database only.
|
|
142
|
+
-O, --online Online mode [default], use this flag to cancel
|
|
143
|
+
'offline: true' in config.
|
|
144
|
+
|
|
145
|
+
<b>DB Maintenance:</b>
|
|
146
|
+
-C, --db-check Check DB version and exit.
|
|
147
|
+
-U, --db-update Migrate to latest DB version and exit.
|
|
148
|
+
-M, --db-migrate VER Migrate to DB version VER and exit.
|
|
149
|
+
--db-create Create database (error if exists).
|
|
150
|
+
--db-reset Drop (if exists) and recreate database. All fetched
|
|
151
|
+
data will be lost.
|
|
152
|
+
|
|
153
|
+
<b>File Arguments:</b>
|
|
154
|
+
‹task› [‹task› ...] One or more names of task files or list files with '@'
|
|
155
|
+
prefix (one task file per line). If task name has not
|
|
156
|
+
extension try to read '‹task›' than '‹task›.inat' than
|
|
157
|
+
'‹task›.rb'.
|
|
158
|
+
</pre>
|
|
159
|
+
|
|
160
|
+
### DSL
|
|
161
|
+
|
|
162
|
+
So, in scripts you can (and should) use specially prepared DSL methods and objects
|
|
163
|
+
responsible for data.
|
|
164
|
+
|
|
165
|
+
First of all, these are **models**, documentation for which can be found at
|
|
166
|
+
<https://inat-get.github.io/inat-get/INatGet/Data/Model.html>. In general, these are simply data
|
|
167
|
+
objects linked to each other. The key ones are probably:
|
|
168
|
+
[`Observation`](https://inat-get.github.io/inat-get/INatGet/Data/Model/Observation.html),
|
|
169
|
+
[`Taxon`](https://inat-get.github.io/inat-get/INatGet/Data/Model/Taxon.html),
|
|
170
|
+
[`Project`](https://inat-get.github.io/inat-get/INatGet/Data/Model/Project.html),
|
|
171
|
+
[`Place`](https://inat-get.github.io/inat-get/INatGet/Data/Model/Place.html)
|
|
172
|
+
and [`User`](https://inat-get.github.io/inat-get/INatGet/Data/Model/User.html).
|
|
173
|
+
|
|
174
|
+
The main work is built on the arithmetic of **datasets** and **lists**. Datasets are data
|
|
175
|
+
selections, manipulations with which happen without real access to the API and DB until
|
|
176
|
+
it becomes necessary. Lists are datasets split by some key field.
|
|
177
|
+
Documentation is at <https://inat-get.github.io/inat-get/INatGet/Data/DSL/Dataset.html>
|
|
178
|
+
for `Dataset` and <https://inat-get.github.io/inat-get/INatGet/Data/DSL/List.html> for `List`.
|
|
179
|
+
Both classes implement the `Enumerable` module, each in its own way...
|
|
180
|
+
|
|
181
|
+
At the beginning of each script we make observation selections (usually) via the `select_observations` method.
|
|
182
|
+
In general, `select_*` and `get_*` methods for different object types are described in the DSL module documentation —
|
|
183
|
+
<https://inat-get.github.io/inat-get/INatGet/Data/DSL.html>. In user scripts this
|
|
184
|
+
module is initially included in the context, so its methods are available directly.
|
|
185
|
+
|
|
186
|
+
*Unfortunately, the fields of selections are not yet documented.* Will be soon.
|
|
187
|
+
|
|
188
|
+
Below are simple examples of user scripts. It is recommended to pay attention
|
|
189
|
+
to "arithmetic" operations: `+`, `-`, `*` and `%`. The last one performs splitting and turns
|
|
190
|
+
a dataset into a list.
|
|
191
|
+
|
|
192
|
+
## Examples
|
|
193
|
+
|
|
194
|
+
### Simple report for a user — [user_stat.rb](share/inat-get/demo/01_user_stat.rb)
|
|
195
|
+
|
|
196
|
+
```ruby
|
|
197
|
+
# Let's make a simple report on taxa observed by the user since the beginning of the year.
|
|
198
|
+
# The report will be output to the current directory with the name user_stat.md (Markdown format)
|
|
199
|
+
|
|
200
|
+
year = today.year
|
|
201
|
+
|
|
202
|
+
user = get_user 'shikhalev' # Here specify the user ID or login, I specified my own
|
|
203
|
+
|
|
204
|
+
# Get observations
|
|
205
|
+
observations = select_observations user: user, observed: time_range(year: year), quality_grade: 'research'
|
|
206
|
+
|
|
207
|
+
by_taxon = observations % :taxon
|
|
208
|
+
|
|
209
|
+
File::open "#{ name }.md", 'w' do |file|
|
|
210
|
+
file.puts '## Report for user ' + user.login + (user.name ? " (#{ user.name })" : '')
|
|
211
|
+
file.puts ''
|
|
212
|
+
by_taxon.each do |ds|
|
|
213
|
+
# Here ds.key is a Taxon object
|
|
214
|
+
file.puts "+ #{ ds.key.common_name } *(#{ ds.key.name })* — #{ ds.count } obs."
|
|
215
|
+
end
|
|
216
|
+
file.puts ''
|
|
217
|
+
file.puts "Total **#{ observations.count }** observations"
|
|
218
|
+
end
|
|
219
|
+
```
|
|
220
|
+
|
|
221
|
+
### Demonstration of list subtraction — [underbound.rb](share/inat-get/demo/02_underfound.rb)
|
|
222
|
+
|
|
223
|
+
```ruby
|
|
224
|
+
# And here we implement the following: for a certain area, find a list of taxa
|
|
225
|
+
# that the given user has not observed (but others have).
|
|
226
|
+
|
|
227
|
+
user = get_user 'shikhalev'
|
|
228
|
+
place = get_place 'artinskiy-gorodskoy-okrug-osm-2023-sv-ru'
|
|
229
|
+
|
|
230
|
+
all_observations = select_observations place: place, quality_grade: 'research', rank: (.. Rank.complex)
|
|
231
|
+
full_list = all_observations % :taxon
|
|
232
|
+
|
|
233
|
+
user_observations = select_observations place: place, quality_grade: "research", rank: (.. Rank.complex), user: user
|
|
234
|
+
user_list = user_observations % :taxon
|
|
235
|
+
|
|
236
|
+
others_list = full_list - user_list
|
|
237
|
+
others_list.sort! { |ds| -ds.count }
|
|
238
|
+
|
|
239
|
+
File::open "#{ name }.md", 'w' do |file|
|
|
240
|
+
file.puts '## Not found by you'
|
|
241
|
+
file.puts ''
|
|
242
|
+
others_list.each do |ds|
|
|
243
|
+
file.puts "+ #{ ds.key.common_name } *(#{ ds.key.name })* — #{ ds.count } obs."
|
|
244
|
+
end
|
|
245
|
+
file.puts ''
|
|
246
|
+
file.puts "Total **#{ others_list.count }** taxa."
|
|
247
|
+
end
|
|
248
|
+
```
|
|
249
|
+
|
|
250
|
+
### List filtering and date range — [newcomers.rb](share/inat-get/demo/03_newcomers.rb)
|
|
251
|
+
|
|
252
|
+
```ruby
|
|
253
|
+
# Newcomers of the previous month. As simple as possible: those who made an observation during
|
|
254
|
+
# the previous month, and registered in it as well. Naturally, within some
|
|
255
|
+
# project, so as not to pull too much.
|
|
256
|
+
|
|
257
|
+
project = get_project 'bioraznoobrazie-rayonov-sverdlovskoy-oblasti'
|
|
258
|
+
|
|
259
|
+
month = today.month - 1
|
|
260
|
+
year = if month == 0
|
|
261
|
+
month = 12
|
|
262
|
+
today.year - 1
|
|
263
|
+
else
|
|
264
|
+
today.year
|
|
265
|
+
end
|
|
266
|
+
|
|
267
|
+
period = time_range year: year, month: month
|
|
268
|
+
observations = select_observations project: project, created: period
|
|
269
|
+
|
|
270
|
+
list = observations % :user
|
|
271
|
+
list.filter! { |ds| period === ds.key.created }
|
|
272
|
+
list.sort! { |ds| ds.key.created }
|
|
273
|
+
|
|
274
|
+
File.open "#{ name }.md", 'w' do |file|
|
|
275
|
+
file.puts "\#\# Newcomers of project «#{ project.title }»"
|
|
276
|
+
file.puts "*#{ period.begin.to_date } — #{ period.end.to_date - 1 }*"
|
|
277
|
+
file.puts ''
|
|
278
|
+
list.each do |ds|
|
|
279
|
+
file.puts "+ #{ ds.key.login } (#{ ds.key.created.to_date }) — #{ ds.count } obs."
|
|
280
|
+
end
|
|
281
|
+
file.puts ''
|
|
282
|
+
file.puts "Total #{ list.count } users"
|
|
283
|
+
end
|
|
284
|
+
```
|
data/bin/inat-get
CHANGED
|
@@ -1,59 +1,8 @@
|
|
|
1
|
-
#!/usr/bin/ruby
|
|
1
|
+
#!/usr/bin/ruby --jit
|
|
2
2
|
|
|
3
|
-
|
|
4
|
-
|
|
3
|
+
require_relative '../lib/inat-get'
|
|
4
|
+
require_relative '../lib/inat-get/app'
|
|
5
5
|
|
|
6
|
-
|
|
6
|
+
app = INatGet::App::Main::instance
|
|
7
7
|
|
|
8
|
-
require 'inat/app/application'
|
|
9
|
-
require 'inat/app/globals'
|
|
10
|
-
require 'inat/data/entity/observation'
|
|
11
|
-
require 'inat/data/entity/request'
|
|
12
|
-
require 'inat/data/db'
|
|
13
|
-
require 'inat/data/query'
|
|
14
|
-
|
|
15
|
-
app = INat::Application.new
|
|
16
8
|
app.run
|
|
17
|
-
# PP::pp Globals.config, $>, 64
|
|
18
|
-
|
|
19
|
-
# puts
|
|
20
|
-
# puts DDL.DDL
|
|
21
|
-
|
|
22
|
-
# PP::pp DB::instance, $>, 64
|
|
23
|
-
|
|
24
|
-
# data = API::query(:observations, user_login: 'shikhalev', month: 11)
|
|
25
|
-
|
|
26
|
-
# PP::pp data.map { |d| Observation::parse(d) }.size, $>, 64
|
|
27
|
-
|
|
28
|
-
# https://www.inaturalist.org/projects/174222
|
|
29
|
-
# https://www.inaturalist.org/places/193592
|
|
30
|
-
# https://www.inaturalist.org/places/193881 (Ачитский район)
|
|
31
|
-
# https://www.inaturalist.org/places/194104 (Богдановичский район)
|
|
32
|
-
# https://www.inaturalist.org/places/194414 (Качканарский район)
|
|
33
|
-
# https://www.inaturalist.org/places/194023 (Берёзовский район)
|
|
34
|
-
# https://api.inaturalist.org/v1/observations?project_id=176067&updated_since=2023-10-31T19:36:29+05:00&per_page=200&order_by=id&order=asc&locale=ru&preferred_place_id=11829
|
|
35
|
-
|
|
36
|
-
# W, [2023-11-02T01:54:41.818021 #8118] WARN -- ‹main›: Some Taxon IDs were not fetched: 48460, 1, 47120, 245097, 47119, 47118, 120474, 342614, 319384, 67599, 495875, 153683, 153680, 367182, 48893, 1252003, 60920!
|
|
37
|
-
# W, [2023-11-02T03:49:58.509031 #8118] WARN -- ‹main›: Some Taxon IDs were not fetched: 1, 47118, 47119, 47120, 48460, 48893, 60920, 67599, 120474, 153680, 153683, 245097, 319384, 342614, 367182, 495875, 1252003!
|
|
38
|
-
|
|
39
|
-
# query = Query::new project_id: 180212
|
|
40
|
-
# PP::pp query.observations.size, $>, 64
|
|
41
|
-
|
|
42
|
-
# require 'inat/data/types/apiquery'
|
|
43
|
-
|
|
44
|
-
# pp Time::new.to_date.to_s
|
|
45
|
-
|
|
46
|
-
# a = ApiQuery::new ololo: 1, lalala: UUID.generate
|
|
47
|
-
|
|
48
|
-
# PP::pp a, $>, 64
|
|
49
|
-
# puts a.to_query
|
|
50
|
-
|
|
51
|
-
# Request::create ''
|
|
52
|
-
|
|
53
|
-
# pp URI::parse('https://api.inaturalist.org/v1/observations?project_id=176067&updated_since=2023-10-31T19:36:29+05:00&per_page=200&order_by=id&order=asc&locale=ru&preferred_place_id=11829')
|
|
54
|
-
|
|
55
|
-
# d = Date::parse '2023-11-01'
|
|
56
|
-
# t = d.to_time
|
|
57
|
-
# i = t.to_i
|
|
58
|
-
|
|
59
|
-
# pp [ d, t, i ]
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'logger'
|
|
4
|
+
require 'singleton'
|
|
5
|
+
|
|
6
|
+
require_relative '../info'
|
|
7
|
+
require_relative 'core/console'
|
|
8
|
+
require_relative 'core/api'
|
|
9
|
+
require_relative 'core/task'
|
|
10
|
+
require_relative 'core/worker'
|
|
11
|
+
|
|
12
|
+
module INatGet::App; end
|
|
13
|
+
|
|
14
|
+
class INatGet::App::Main
|
|
15
|
+
|
|
16
|
+
include Singleton
|
|
17
|
+
|
|
18
|
+
def initialize
|
|
19
|
+
@config = INatGet::App::Setup::config!
|
|
20
|
+
if @config[:tasks].nil?
|
|
21
|
+
warn "❌ No tasks specified!"
|
|
22
|
+
exit Errno::ECHILD::Errno
|
|
23
|
+
end
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
def run
|
|
27
|
+
console_socket = @config.dig :socket, :console
|
|
28
|
+
api_socket = @config.dig :socket, :api
|
|
29
|
+
check_sockets! api_socket, console_socket
|
|
30
|
+
|
|
31
|
+
INatGet::App::Maintenance::db_check @config, true
|
|
32
|
+
|
|
33
|
+
console = INatGet::App::Server::Console::create console_socket
|
|
34
|
+
api = nil
|
|
35
|
+
api = INatGet::App::Server::API::create api_socket, console: console if !@config[:offline]
|
|
36
|
+
|
|
37
|
+
tasks = @config[:tasks].map { |path| INatGet::App::Task::new path, @config, console: console, api: api }
|
|
38
|
+
Process::warmup
|
|
39
|
+
INatGet::App::Worker::enqueue @config, *tasks, console: console, api: api
|
|
40
|
+
console.quit
|
|
41
|
+
api.quit if api
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
private
|
|
45
|
+
|
|
46
|
+
def check_sockets! socket, socket2
|
|
47
|
+
if File.exist?(socket)
|
|
48
|
+
if socket_alive?(socket)
|
|
49
|
+
warn "❌ API Socket already exists!"
|
|
50
|
+
exit Errno::EEXIST::Errno
|
|
51
|
+
else
|
|
52
|
+
File.delete socket
|
|
53
|
+
end
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
if File.exist?(socket2)
|
|
57
|
+
File.delete socket2
|
|
58
|
+
end
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
def socket_alive? socket
|
|
62
|
+
return false unless File.socket?(socket)
|
|
63
|
+
INatGet::App::Server::used? socket
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
end
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'faraday'
|
|
4
|
+
require 'faraday/retry'
|
|
5
|
+
require 'is-duration'
|
|
6
|
+
|
|
7
|
+
require_relative 'server'
|
|
8
|
+
require_relative 'console_logger'
|
|
9
|
+
|
|
10
|
+
class INatGet::App::Server::API < INatGet::App::Server
|
|
11
|
+
|
|
12
|
+
def initialize socket_path, **params
|
|
13
|
+
@console = params.delete :console
|
|
14
|
+
@logger = ::INatGet::App::ConsoleLogger::new @console, progname: 'API'
|
|
15
|
+
super(socket_path, **params)
|
|
16
|
+
@config = INatGet::App::Setup::config
|
|
17
|
+
@delay = IS::Duration::parse @config.dig(:api, :delay)
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
private
|
|
21
|
+
|
|
22
|
+
def get query, **opts
|
|
23
|
+
endpoint = @config.dig(:api, :root) + query[:endpoint].to_s
|
|
24
|
+
timepoint = Time::now
|
|
25
|
+
if @last_request
|
|
26
|
+
delta = timepoint - @last_request
|
|
27
|
+
sleep (@delay - delta) if delta < @delay
|
|
28
|
+
end
|
|
29
|
+
@last_request = timepoint
|
|
30
|
+
response = faraday.get(endpoint) do |rq|
|
|
31
|
+
rq.params[:per_page] = @config.dig(:api, :pager)
|
|
32
|
+
rq.params.compact!
|
|
33
|
+
rq.params.merge! query[:query]
|
|
34
|
+
rq.headers["User-Agent"] = "iNatGet v#{INatGet::Info::VERSION} (#{ INatGet::Info::VERSION_ALIAS })"
|
|
35
|
+
end
|
|
36
|
+
if response.success?
|
|
37
|
+
begin
|
|
38
|
+
data = JSON.parse response.body, symbolize_names: true
|
|
39
|
+
return data.freeze
|
|
40
|
+
rescue => e
|
|
41
|
+
@logger.error "Error while parsing: #{e.message}"
|
|
42
|
+
return { status: :error, error: e.message }.freeze
|
|
43
|
+
end
|
|
44
|
+
else
|
|
45
|
+
@logger.error "Error in response: #{response.status}"
|
|
46
|
+
return { status: :error, error: response.status }.freeze
|
|
47
|
+
end
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
def faraday
|
|
51
|
+
# tmp_logger = ::Logger::new 'common.log', level: :info
|
|
52
|
+
@faraday ||= Faraday::new do |f|
|
|
53
|
+
f.request :retry,
|
|
54
|
+
max: @config.dig(:api, :retry, :max),
|
|
55
|
+
interval: IS::Duration::parse(@config.dig(:api, :retry, :interval)),
|
|
56
|
+
interval_randomness: @config.dig(:api, :retry, :randomness),
|
|
57
|
+
backoff_factor: @config.dig(:api, :retry, :backoff),
|
|
58
|
+
exceptions: [Faraday::TimeoutError, Faraday::ConnectionFailed, Faraday::SSLError, Faraday::ClientError]
|
|
59
|
+
f.request :url_encoded
|
|
60
|
+
# f.response :logger, tmp_logger, bodies: true, headers: true
|
|
61
|
+
f.adapter Faraday::default_adapter
|
|
62
|
+
end
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
end
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'logger'
|
|
4
|
+
|
|
5
|
+
require 'is-term/statustable'
|
|
6
|
+
require 'is-term/formats'
|
|
7
|
+
require 'is-term/functions'
|
|
8
|
+
|
|
9
|
+
require_relative 'server'
|
|
10
|
+
|
|
11
|
+
class INatGet::App::Server::Console < INatGet::App::Server
|
|
12
|
+
|
|
13
|
+
def initialize socket_path, **params
|
|
14
|
+
super(socket_path, **params)
|
|
15
|
+
@table = IS::Term::StatusTable::instance
|
|
16
|
+
@table.configure do
|
|
17
|
+
column :icon, func: lambda { |row| row[:_active] ? "\e[1m[ ]" : "[✔]" }
|
|
18
|
+
separator ' '
|
|
19
|
+
column :_sender_pid, id: true, align: :right
|
|
20
|
+
separator ' '
|
|
21
|
+
column :name
|
|
22
|
+
separator ' '
|
|
23
|
+
column :status, align: :center
|
|
24
|
+
separator ' '
|
|
25
|
+
column :current, align: :right, summary: :current
|
|
26
|
+
separator ' of '
|
|
27
|
+
column :total, align: :left, summary: :total
|
|
28
|
+
separator ' '
|
|
29
|
+
column :percent, align: :right, func: :percent, format: '%d%%', summary: :percent
|
|
30
|
+
separator ' '
|
|
31
|
+
column :estimated, func: :estimated, align: :right, format: :duration, summary: :elapsed
|
|
32
|
+
separator ' '
|
|
33
|
+
column :speed, func: :speed, format: '%.2f r/s', align: :right, summary: :speed
|
|
34
|
+
separator ' '
|
|
35
|
+
column :message, summary: :value
|
|
36
|
+
|
|
37
|
+
summary true, message: ''
|
|
38
|
+
end
|
|
39
|
+
if @config.dig(:logs, :file, :enable)
|
|
40
|
+
filename = @config.dig(:logs, :file, :enable)
|
|
41
|
+
@sys_logger = Logger::new filename, 'daily', 7, level: @config.dig(:logs, :file, :sys)
|
|
42
|
+
@api_logger = Logger::new filename, 'daily', 7, level: @config.dig(:logs, :file, :api)
|
|
43
|
+
@wrk_logger = Logger::new filename, 'daily', 7, level: @config.dig(:logs, :file, :wrk)
|
|
44
|
+
end
|
|
45
|
+
@log_levels = {
|
|
46
|
+
sys: Logger::Severity::coerce(@config.dig(:logs, :screen, :sys)),
|
|
47
|
+
api: Logger::Severity::coerce(@config.dig(:logs, :screen, :api)),
|
|
48
|
+
wrk: Logger::Severity::coerce(@config.dig(:logs, :screen, :wrk))
|
|
49
|
+
}
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
def register **opts
|
|
53
|
+
@table.append(**opts)
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
def update **opts
|
|
57
|
+
@table.update(**opts)
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
def log severity, message, progname, **opts
|
|
61
|
+
key = prog_key progname
|
|
62
|
+
pid = opts[:_sender_pid]
|
|
63
|
+
console_log severity, message, progname, key, pid
|
|
64
|
+
file_log severity, message, progname, key if @config.dig(:logs, :file, :enable)
|
|
65
|
+
true
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
private
|
|
69
|
+
|
|
70
|
+
def prog_key progname
|
|
71
|
+
if progname.nil? || progname == ''
|
|
72
|
+
:sys
|
|
73
|
+
elsif progname == 'SYS' || progname == 'API'
|
|
74
|
+
progname.downcase.to_sym
|
|
75
|
+
else
|
|
76
|
+
:wrk
|
|
77
|
+
end
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
def file_log severity, message, progname, prog_key
|
|
81
|
+
case prog_key
|
|
82
|
+
when :api
|
|
83
|
+
@api_logger.log severity, message, progname
|
|
84
|
+
when :wrk
|
|
85
|
+
@wrk_logger.log severity, message, progname
|
|
86
|
+
else
|
|
87
|
+
@sys_logger.log severity, message, progname
|
|
88
|
+
end
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
SEV_ICON = {
|
|
92
|
+
Logger::Severity::DEBUG => '📓',
|
|
93
|
+
Logger::Severity::INFO => '📢',
|
|
94
|
+
Logger::Severity::WARN => '🔔',
|
|
95
|
+
Logger::Severity::ERROR => '🚨',
|
|
96
|
+
Logger::Severity::FATAL => '❌'
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
def console_log severity, message, progname, prog_key, pid
|
|
100
|
+
if severity >= @log_levels[prog_key]
|
|
101
|
+
if prog_key == :wrk
|
|
102
|
+
@table.update _sender_pid: pid, message: "#{ SEV_ICON[severity] } #{ message }"
|
|
103
|
+
else
|
|
104
|
+
@table.summary message: "#{ SEV_ICON[severity] } [#{ progname }] #{ message }"
|
|
105
|
+
end
|
|
106
|
+
end
|
|
107
|
+
end
|
|
108
|
+
|
|
109
|
+
end
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'logger'
|
|
4
|
+
|
|
5
|
+
require_relative '../../info'
|
|
6
|
+
|
|
7
|
+
module INatGet::App; end
|
|
8
|
+
|
|
9
|
+
class INatGet::App::ConsoleLogger < Logger
|
|
10
|
+
|
|
11
|
+
def initialize console, **options
|
|
12
|
+
@console = console
|
|
13
|
+
super(nil, **options)
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
def add severity, msg = nil, progname = nil
|
|
17
|
+
if block_given?
|
|
18
|
+
msg = yield
|
|
19
|
+
elsif msg.nil?
|
|
20
|
+
msg = progname
|
|
21
|
+
progname = nil
|
|
22
|
+
end
|
|
23
|
+
return if msg.nil?
|
|
24
|
+
@console.log severity, msg, (progname || self.progname)
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
end
|