@serve.zone/gitops 2.13.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (258) hide show
  1. package/.smartconfig.json +114 -0
  2. package/binary/gitops.ts +4 -0
  3. package/changelog.md +185 -0
  4. package/cli.child.js +4 -0
  5. package/cli.js +4 -0
  6. package/cli.ts.js +5 -0
  7. package/deno.json +10 -0
  8. package/dist_serve/bundle.js +36362 -0
  9. package/dist_serve/index.html +33 -0
  10. package/dist_ts/00_commitinfo_data.d.ts +8 -0
  11. package/dist_ts/00_commitinfo_data.js +9 -0
  12. package/dist_ts/cache/classes.cache.cleaner.d.ts +23 -0
  13. package/dist_ts/cache/classes.cache.cleaner.js +61 -0
  14. package/dist_ts/cache/classes.cached.document.d.ts +30 -0
  15. package/dist_ts/cache/classes.cached.document.js +101 -0
  16. package/dist_ts/cache/classes.cachedb.d.ts +22 -0
  17. package/dist_ts/cache/classes.cachedb.js +58 -0
  18. package/dist_ts/cache/classes.secrets.scan.service.d.ts +51 -0
  19. package/dist_ts/cache/classes.secrets.scan.service.js +237 -0
  20. package/dist_ts/cache/documents/classes.cached.project.d.ts +13 -0
  21. package/dist_ts/cache/documents/classes.cached.project.js +101 -0
  22. package/dist_ts/cache/documents/classes.cached.secret.d.ts +24 -0
  23. package/dist_ts/cache/documents/classes.cached.secret.js +158 -0
  24. package/dist_ts/cache/documents/index.d.ts +2 -0
  25. package/dist_ts/cache/documents/index.js +3 -0
  26. package/dist_ts/cache/index.d.ts +7 -0
  27. package/dist_ts/cache/index.js +6 -0
  28. package/dist_ts/classes/actionlog.d.ts +19 -0
  29. package/dist_ts/classes/actionlog.js +44 -0
  30. package/dist_ts/classes/connectionmanager.d.ts +57 -0
  31. package/dist_ts/classes/connectionmanager.js +247 -0
  32. package/dist_ts/classes/gitopsapp.d.ts +30 -0
  33. package/dist_ts/classes/gitopsapp.js +101 -0
  34. package/dist_ts/classes/jobmanager.d.ts +47 -0
  35. package/dist_ts/classes/jobmanager.js +301 -0
  36. package/dist_ts/classes/jobrunners/autobookstackdocs.runner.d.ts +29 -0
  37. package/dist_ts/classes/jobrunners/autobookstackdocs.runner.js +361 -0
  38. package/dist_ts/classes/jobrunners/base.jobrunner.d.ts +14 -0
  39. package/dist_ts/classes/jobrunners/base.jobrunner.js +3 -0
  40. package/dist_ts/classes/jobrunners/index.d.ts +5 -0
  41. package/dist_ts/classes/jobrunners/index.js +14 -0
  42. package/dist_ts/classes/managedsecrets.manager.d.ts +47 -0
  43. package/dist_ts/classes/managedsecrets.manager.js +247 -0
  44. package/dist_ts/classes/syncmanager.d.ts +189 -0
  45. package/dist_ts/classes/syncmanager.js +1787 -0
  46. package/dist_ts/index.d.ts +6 -0
  47. package/dist_ts/index.js +32 -0
  48. package/dist_ts/logging.d.ts +49 -0
  49. package/dist_ts/logging.js +134 -0
  50. package/dist_ts/opsserver/classes.opsserver.d.ts +25 -0
  51. package/dist_ts/opsserver/classes.opsserver.js +70 -0
  52. package/dist_ts/opsserver/handlers/actionlog.handler.d.ts +9 -0
  53. package/dist_ts/opsserver/handlers/actionlog.handler.js +24 -0
  54. package/dist_ts/opsserver/handlers/actions.handler.d.ts +9 -0
  55. package/dist_ts/opsserver/handlers/actions.handler.js +38 -0
  56. package/dist_ts/opsserver/handlers/admin.handler.d.ts +19 -0
  57. package/dist_ts/opsserver/handlers/admin.handler.js +96 -0
  58. package/dist_ts/opsserver/handlers/connections.handler.d.ts +10 -0
  59. package/dist_ts/opsserver/handlers/connections.handler.js +109 -0
  60. package/dist_ts/opsserver/handlers/groups.handler.d.ts +9 -0
  61. package/dist_ts/opsserver/handlers/groups.handler.js +24 -0
  62. package/dist_ts/opsserver/handlers/index.d.ts +13 -0
  63. package/dist_ts/opsserver/handlers/index.js +14 -0
  64. package/dist_ts/opsserver/handlers/jobs.handler.d.ts +16 -0
  65. package/dist_ts/opsserver/handlers/jobs.handler.js +146 -0
  66. package/dist_ts/opsserver/handlers/logs.handler.d.ts +9 -0
  67. package/dist_ts/opsserver/handlers/logs.handler.js +21 -0
  68. package/dist_ts/opsserver/handlers/managedsecrets.handler.d.ts +11 -0
  69. package/dist_ts/opsserver/handlers/managedsecrets.handler.js +110 -0
  70. package/dist_ts/opsserver/handlers/pipelines.handler.d.ts +31 -0
  71. package/dist_ts/opsserver/handlers/pipelines.handler.js +204 -0
  72. package/dist_ts/opsserver/handlers/projects.handler.d.ts +9 -0
  73. package/dist_ts/opsserver/handlers/projects.handler.js +24 -0
  74. package/dist_ts/opsserver/handlers/secrets.handler.d.ts +10 -0
  75. package/dist_ts/opsserver/handlers/secrets.handler.js +171 -0
  76. package/dist_ts/opsserver/handlers/sync.handler.d.ts +16 -0
  77. package/dist_ts/opsserver/handlers/sync.handler.js +166 -0
  78. package/dist_ts/opsserver/handlers/webhook.handler.d.ts +7 -0
  79. package/dist_ts/opsserver/handlers/webhook.handler.js +55 -0
  80. package/dist_ts/opsserver/helpers/guards.d.ts +5 -0
  81. package/dist_ts/opsserver/helpers/guards.js +12 -0
  82. package/dist_ts/opsserver/index.d.ts +1 -0
  83. package/dist_ts/opsserver/index.js +2 -0
  84. package/dist_ts/paths.d.ts +9 -0
  85. package/dist_ts/paths.js +13 -0
  86. package/dist_ts/plugins.d.ts +25 -0
  87. package/dist_ts/plugins.js +32 -0
  88. package/dist_ts/providers/classes.baseprovider.d.ts +51 -0
  89. package/dist_ts/providers/classes.baseprovider.js +17 -0
  90. package/dist_ts/providers/classes.giteaprovider.d.ts +40 -0
  91. package/dist_ts/providers/classes.giteaprovider.js +224 -0
  92. package/dist_ts/providers/classes.gitlabprovider.d.ts +39 -0
  93. package/dist_ts/providers/classes.gitlabprovider.js +207 -0
  94. package/dist_ts/providers/index.d.ts +3 -0
  95. package/dist_ts/providers/index.js +4 -0
  96. package/dist_ts/storage/classes.storagemanager.d.ts +33 -0
  97. package/dist_ts/storage/classes.storagemanager.js +135 -0
  98. package/dist_ts/storage/index.d.ts +2 -0
  99. package/dist_ts/storage/index.js +2 -0
  100. package/dist_ts/timers.d.ts +4 -0
  101. package/dist_ts/timers.js +24 -0
  102. package/dist_ts_bundled/bundle.d.ts +4 -0
  103. package/dist_ts_bundled/bundle.js +12 -0
  104. package/dist_ts_interfaces/data/actionlog.d.ts +12 -0
  105. package/dist_ts_interfaces/data/actionlog.js +2 -0
  106. package/dist_ts_interfaces/data/branch.d.ts +8 -0
  107. package/dist_ts_interfaces/data/branch.js +2 -0
  108. package/dist_ts_interfaces/data/connection.d.ts +12 -0
  109. package/dist_ts_interfaces/data/connection.js +2 -0
  110. package/dist_ts_interfaces/data/group.d.ts +10 -0
  111. package/dist_ts_interfaces/data/group.js +2 -0
  112. package/dist_ts_interfaces/data/identity.d.ts +7 -0
  113. package/dist_ts_interfaces/data/identity.js +2 -0
  114. package/dist_ts_interfaces/data/index.d.ts +11 -0
  115. package/dist_ts_interfaces/data/index.js +12 -0
  116. package/dist_ts_interfaces/data/job.d.ts +37 -0
  117. package/dist_ts_interfaces/data/job.js +2 -0
  118. package/dist_ts_interfaces/data/managedsecret.d.ts +37 -0
  119. package/dist_ts_interfaces/data/managedsecret.js +2 -0
  120. package/dist_ts_interfaces/data/pipeline.d.ts +22 -0
  121. package/dist_ts_interfaces/data/pipeline.js +2 -0
  122. package/dist_ts_interfaces/data/project.d.ts +12 -0
  123. package/dist_ts_interfaces/data/project.js +2 -0
  124. package/dist_ts_interfaces/data/secret.d.ts +11 -0
  125. package/dist_ts_interfaces/data/secret.js +2 -0
  126. package/dist_ts_interfaces/data/sync.d.ts +34 -0
  127. package/dist_ts_interfaces/data/sync.js +2 -0
  128. package/dist_ts_interfaces/index.d.ts +5 -0
  129. package/dist_ts_interfaces/index.js +8 -0
  130. package/dist_ts_interfaces/plugins.d.ts +2 -0
  131. package/dist_ts_interfaces/plugins.js +4 -0
  132. package/dist_ts_interfaces/requests/actionlog.d.ts +15 -0
  133. package/dist_ts_interfaces/requests/actionlog.js +3 -0
  134. package/dist_ts_interfaces/requests/actions.d.ts +31 -0
  135. package/dist_ts_interfaces/requests/actions.js +3 -0
  136. package/dist_ts_interfaces/requests/admin.d.ts +31 -0
  137. package/dist_ts_interfaces/requests/admin.js +3 -0
  138. package/dist_ts_interfaces/requests/connections.d.ts +71 -0
  139. package/dist_ts_interfaces/requests/connections.js +3 -0
  140. package/dist_ts_interfaces/requests/groups.d.ts +14 -0
  141. package/dist_ts_interfaces/requests/groups.js +3 -0
  142. package/dist_ts_interfaces/requests/index.d.ts +13 -0
  143. package/dist_ts_interfaces/requests/index.js +14 -0
  144. package/dist_ts_interfaces/requests/jobs.d.ts +86 -0
  145. package/dist_ts_interfaces/requests/jobs.js +3 -0
  146. package/dist_ts_interfaces/requests/logs.d.ts +14 -0
  147. package/dist_ts_interfaces/requests/logs.js +3 -0
  148. package/dist_ts_interfaces/requests/managedsecrets.d.ts +84 -0
  149. package/dist_ts_interfaces/requests/managedsecrets.js +3 -0
  150. package/dist_ts_interfaces/requests/pipelines.d.ts +55 -0
  151. package/dist_ts_interfaces/requests/pipelines.js +3 -0
  152. package/dist_ts_interfaces/requests/projects.d.ts +14 -0
  153. package/dist_ts_interfaces/requests/projects.js +3 -0
  154. package/dist_ts_interfaces/requests/secrets.d.ts +72 -0
  155. package/dist_ts_interfaces/requests/secrets.js +3 -0
  156. package/dist_ts_interfaces/requests/sync.d.ts +120 -0
  157. package/dist_ts_interfaces/requests/sync.js +3 -0
  158. package/dist_ts_interfaces/requests/webhook.d.ts +13 -0
  159. package/dist_ts_interfaces/requests/webhook.js +3 -0
  160. package/license +21 -0
  161. package/package.json +81 -0
  162. package/readme.md +177 -0
  163. package/readme.todo.md +3 -0
  164. package/ts/00_commitinfo_data.ts +8 -0
  165. package/ts/cache/classes.cache.cleaner.ts +69 -0
  166. package/ts/cache/classes.cached.document.ts +57 -0
  167. package/ts/cache/classes.cachedb.ts +72 -0
  168. package/ts/cache/classes.secrets.scan.service.ts +267 -0
  169. package/ts/cache/documents/classes.cached.project.ts +32 -0
  170. package/ts/cache/documents/classes.cached.secret.ts +81 -0
  171. package/ts/cache/documents/index.ts +2 -0
  172. package/ts/cache/index.ts +7 -0
  173. package/ts/classes/actionlog.ts +57 -0
  174. package/ts/classes/connectionmanager.ts +263 -0
  175. package/ts/classes/gitopsapp.ts +128 -0
  176. package/ts/classes/jobmanager.ts +337 -0
  177. package/ts/classes/jobrunners/autobookstackdocs.runner.ts +435 -0
  178. package/ts/classes/jobrunners/base.jobrunner.ts +16 -0
  179. package/ts/classes/jobrunners/index.ts +17 -0
  180. package/ts/classes/managedsecrets.manager.ts +322 -0
  181. package/ts/classes/syncmanager.ts +2117 -0
  182. package/ts/index.ts +37 -0
  183. package/ts/logging.ts +162 -0
  184. package/ts/opsserver/classes.opsserver.ts +86 -0
  185. package/ts/opsserver/handlers/actionlog.handler.ts +30 -0
  186. package/ts/opsserver/handlers/actions.handler.ts +50 -0
  187. package/ts/opsserver/handlers/admin.handler.ts +122 -0
  188. package/ts/opsserver/handlers/connections.handler.ts +162 -0
  189. package/ts/opsserver/handlers/groups.handler.ts +32 -0
  190. package/ts/opsserver/handlers/index.ts +13 -0
  191. package/ts/opsserver/handlers/jobs.handler.ts +189 -0
  192. package/ts/opsserver/handlers/logs.handler.ts +29 -0
  193. package/ts/opsserver/handlers/managedsecrets.handler.ts +158 -0
  194. package/ts/opsserver/handlers/pipelines.handler.ts +281 -0
  195. package/ts/opsserver/handlers/projects.handler.ts +32 -0
  196. package/ts/opsserver/handlers/secrets.handler.ts +224 -0
  197. package/ts/opsserver/handlers/sync.handler.ts +224 -0
  198. package/ts/opsserver/handlers/webhook.handler.ts +62 -0
  199. package/ts/opsserver/helpers/guards.ts +16 -0
  200. package/ts/opsserver/index.ts +1 -0
  201. package/ts/paths.ts +19 -0
  202. package/ts/plugins.ts +38 -0
  203. package/ts/providers/classes.baseprovider.ts +99 -0
  204. package/ts/providers/classes.giteaprovider.ts +279 -0
  205. package/ts/providers/classes.gitlabprovider.ts +265 -0
  206. package/ts/providers/index.ts +3 -0
  207. package/ts/storage/classes.storagemanager.ts +144 -0
  208. package/ts/storage/index.ts +2 -0
  209. package/ts/timers.ts +34 -0
  210. package/ts_interfaces/data/actionlog.ts +13 -0
  211. package/ts_interfaces/data/branch.ts +9 -0
  212. package/ts_interfaces/data/connection.ts +13 -0
  213. package/ts_interfaces/data/group.ts +10 -0
  214. package/ts_interfaces/data/identity.ts +7 -0
  215. package/ts_interfaces/data/index.ts +11 -0
  216. package/ts_interfaces/data/job.ts +42 -0
  217. package/ts_interfaces/data/managedsecret.ts +41 -0
  218. package/ts_interfaces/data/pipeline.ts +32 -0
  219. package/ts_interfaces/data/project.ts +12 -0
  220. package/ts_interfaces/data/secret.ts +11 -0
  221. package/ts_interfaces/data/sync.ts +37 -0
  222. package/ts_interfaces/index.ts +9 -0
  223. package/ts_interfaces/plugins.ts +6 -0
  224. package/ts_interfaces/requests/actionlog.ts +19 -0
  225. package/ts_interfaces/requests/actions.ts +39 -0
  226. package/ts_interfaces/requests/admin.ts +43 -0
  227. package/ts_interfaces/requests/connections.ts +95 -0
  228. package/ts_interfaces/requests/groups.ts +18 -0
  229. package/ts_interfaces/requests/index.ts +13 -0
  230. package/ts_interfaces/requests/jobs.ts +118 -0
  231. package/ts_interfaces/requests/logs.ts +18 -0
  232. package/ts_interfaces/requests/managedsecrets.ts +112 -0
  233. package/ts_interfaces/requests/pipelines.ts +71 -0
  234. package/ts_interfaces/requests/projects.ts +18 -0
  235. package/ts_interfaces/requests/secrets.ts +92 -0
  236. package/ts_interfaces/requests/sync.ts +157 -0
  237. package/ts_interfaces/requests/webhook.ts +18 -0
  238. package/ts_web/00_commitinfo_data.ts +8 -0
  239. package/ts_web/appstate.ts +1251 -0
  240. package/ts_web/elements/gitops-dashboard.ts +350 -0
  241. package/ts_web/elements/index.ts +10 -0
  242. package/ts_web/elements/shared/css.ts +29 -0
  243. package/ts_web/elements/shared/index.ts +1 -0
  244. package/ts_web/elements/views/actionlog/index.ts +101 -0
  245. package/ts_web/elements/views/actions/index.ts +209 -0
  246. package/ts_web/elements/views/buildlog/index.ts +196 -0
  247. package/ts_web/elements/views/connections/index.ts +260 -0
  248. package/ts_web/elements/views/groups/index.ts +134 -0
  249. package/ts_web/elements/views/jobs/index.ts +424 -0
  250. package/ts_web/elements/views/managedsecrets/index.ts +502 -0
  251. package/ts_web/elements/views/overview/index.ts +86 -0
  252. package/ts_web/elements/views/pipelines/index.ts +561 -0
  253. package/ts_web/elements/views/projects/index.ts +149 -0
  254. package/ts_web/elements/views/secrets/index.ts +310 -0
  255. package/ts_web/elements/views/sync/index.ts +512 -0
  256. package/ts_web/index.ts +7 -0
  257. package/ts_web/plugins.ts +15 -0
  258. package/tsconfig.json +15 -0
package/readme.md ADDED
@@ -0,0 +1,177 @@
1
+ # @serve.zone/gitops
2
+
3
+ GitOps is a self-hosted operations dashboard for Gitea and GitLab. It connects provider instances, centralizes CI/CD secret visibility, mirrors repositories, watches pipelines, streams job logs, and can publish repository documentation into BookStack from one Node.js-powered web app.
4
+
5
+ ## Issue Reporting and Security
6
+
7
+ For reporting bugs, issues, or security vulnerabilities, please visit [community.foss.global/](https://community.foss.global/). This is the central community hub for all issue reporting. Developers who sign and comply with our contribution agreement and go through identification can also get a [code.foss.global/](https://code.foss.global/) account to submit Pull Requests directly.
8
+
9
+ ## What It Does
10
+
11
+ - Connects to Gitea and GitLab through provider adapters with a shared project, group, secret, pipeline, and file-content interface.
12
+ - Stores provider tokens and BookStack job tokens through `@push.rocks/smartsecret`, using OS keychain support where available and an encrypted fallback otherwise.
13
+ - Serves a bundled Lit/dees-catalog single page app through `@api.global/typedserver` with TypedRequest handlers for all backend actions.
14
+ - Scans project and group CI/CD secrets into an embedded SmartMongo cache and refreshes that cache on startup and every 24 hours.
15
+ - Mirrors repositories from a source connection to a target connection with group offset support, metadata sync, stale repository handling, and live sync logs.
16
+ - Manages reusable secret definitions that can be pushed to selected project or group targets.
17
+ - Runs scheduled jobs, currently focused on syncing `readme.md`, `changelog.md`, and `docs/*.md` content into BookStack shelves, books, and pages.
18
+ - Receives provider webhooks at `POST /webhook/:connectionId` and broadcasts events to connected browser clients.
19
+
20
+ ## Runtime Shape
21
+
22
+ GitOps is a Node.js service packaged as an npm module. Deno is used only for compiling self-contained binary targets through `@git.zone/tsdeno`.
23
+
24
+ | Area | Implementation |
25
+ | --- | --- |
26
+ | Entry point | `cli.js`, default command `server` |
27
+ | Backend | `ts/`, centered around `GitopsApp` |
28
+ | Web server | `ts/opsserver/`, TypedServer plus TypedRequest handlers |
29
+ | Shared contracts | `ts_interfaces/` data and request interfaces |
30
+ | Frontend | `ts_web/`, Lit web components and smartstate |
31
+ | Build output | `dist_ts/`, `dist_ts_interfaces/`, and `dist_serve/` generated by `tsbuild` and `tsbundle` |
32
+ | Persistent storage | `~/.serve.zone/gitops/storage/` |
33
+ | Cache database | `~/.serve.zone/gitops/tsmdb/` |
34
+
35
+ ## Installation and Usage
36
+
37
+ Install the package through pnpm and start the public `gitops` binary:
38
+
39
+ ```bash
40
+ pnpm add -g @serve.zone/gitops
41
+ GITOPS_PORT=3000 gitops server
42
+ ```
43
+
44
+ For local source runs, use the checked-in Node entrypoint:
45
+
46
+ ```bash
47
+ node ./cli.js server
48
+ GITOPS_PORT=3001 node ./cli.js server
49
+ ```
50
+
51
+ The package exports the application API and shared request/data contracts for typed integrations:
52
+
53
+ ```typescript
54
+ import { GitopsApp } from '@serve.zone/gitops';
55
+ import type * as gitopsInterfaces from '@serve.zone/gitops/interfaces';
56
+ ```
57
+
58
+ ## Key Components
59
+
60
+ | Component | Role |
61
+ | --- | --- |
62
+ | `GitopsApp` | Starts storage, provider connections, cache, sync, jobs, managed secrets, scans, and the OpsServer. |
63
+ | `ConnectionManager` | Persists Gitea/GitLab connections, migrates old plaintext tokens into SmartSecret, and background-checks connection health. |
64
+ | `GiteaProvider` / `GitLabProvider` | Provider-specific adapters behind a shared `BaseProvider` interface. |
65
+ | `SecretsScanService` | Fetches project and group secrets in batches and upserts them into cached SmartData documents. |
66
+ | `SyncManager` | Mirrors repositories with bare git clones, source/target path mapping, deletion safeguards, branch/default-branch handling, and metadata sync. |
67
+ | `ManagedSecretsManager` | Stores define-once secret records and pushes them to selected project or group targets. |
68
+ | `JobManager` | Stores scheduled jobs, starts interval timers, masks persisted secrets, and dispatches job execution to registered runners. |
69
+ | `AutoBookstackDocsRunner` | Maps git groups to BookStack shelves, repos to books, and markdown files to pages with content hashes to avoid noisy updates. |
70
+
71
+ ## Server Configuration
72
+
73
+ | Variable | Default | Purpose |
74
+ | --- | --- | --- |
75
+ | `GITOPS_PORT` | `3000` | HTTP and WebSocket server port. |
76
+ | `GITOPS_ADMIN_USERNAME` | `admin` | Dashboard login username. |
77
+ | `GITOPS_ADMIN_PASSWORD` | `admin` | Dashboard login password. |
78
+
79
+ Provider connections, sync configs, jobs, managed secrets, and action logs are persisted as JSON records below `~/.serve.zone/gitops/storage/`. Cached projects and secrets live in the embedded SmartMongo directory below `~/.serve.zone/gitops/tsmdb/`.
80
+
81
+ ## API Surface
82
+
83
+ The backend exposes TypedRequest methods through `/typedrequest` and a custom webhook route. The important handler groups are:
84
+
85
+ | Handler | Examples |
86
+ | --- | --- |
87
+ | Admin | `adminLogin`, `adminLogout`, `verifyIdentity` |
88
+ | Connections | `getConnections`, `createConnection`, `updateConnection`, `testConnection`, `pauseConnection`, `deleteConnection` |
89
+ | Projects and Groups | Provider browsing with search/pagination style request contracts |
90
+ | Secrets | Cached and direct project/group secret CRUD |
91
+ | Managed Secrets | Definition CRUD plus push-to-target operations |
92
+ | Pipelines and Logs | Pipeline listing, jobs, retry/cancel, and raw job log fetches |
93
+ | Sync | Sync config CRUD, preview, trigger, status, and logs |
94
+ | Jobs | Scheduled job CRUD, trigger, pause/resume, and logs |
95
+ | Webhooks | `POST /webhook/:connectionId` for provider event fan-out |
96
+
97
+ ## Auto BookStack Documentation
98
+
99
+ The current scheduled job type is `autobookstackdocs`. It reads markdown documentation from connected Git providers and publishes it into BookStack.
100
+
101
+ | Git source | BookStack target |
102
+ | --- | --- |
103
+ | Group or organization | Shelf |
104
+ | Repository | Book named `@group/repo` |
105
+ | `readme.md` or `changelog.md` | Page named `<file> for @group/repo` |
106
+ | `docs/*.md` | Page named `<path> for @group/repo` |
107
+
108
+ Notable behavior from the runner implementation:
109
+
110
+ - SHA-256 hashes are persisted per job so unchanged pages are skipped.
111
+ - Optional delete propagation can remove stale shelves, books, and pages.
112
+ - Optional visibility sync can restrict private/internal repos in BookStack.
113
+ - Tags are collected from git topics and `package.json` keywords.
114
+ - Root headings that duplicate the book or page context are stripped before publication.
115
+
116
+ ## Development
117
+
118
+ ```bash
119
+ pnpm install
120
+ pnpm build
121
+ pnpm test
122
+ pnpm run startTs
123
+ ```
124
+
125
+ Useful direct commands:
126
+
127
+ ```bash
128
+ node ./cli.js server
129
+ GITOPS_PORT=3001 node ./cli.js server
130
+ pnpm run build:binary
131
+ pnpm run build:docker
132
+ pnpm run watch
133
+ ```
134
+
135
+ ## Project Map
136
+
137
+ ```text
138
+ gitops/
139
+ ├── cli.js
140
+ ├── cli.ts.js
141
+ ├── binary/
142
+ ├── deno.json
143
+ ├── package.json
144
+ ├── tsconfig.json
145
+ ├── html/
146
+ ├── ts/
147
+ │ ├── classes/
148
+ │ ├── cache/
149
+ │ ├── opsserver/
150
+ │ ├── providers/
151
+ │ └── storage/
152
+ ├── ts_interfaces/
153
+ ├── ts_web/
154
+ ├── dist_serve/
155
+ └── test/
156
+ ```
157
+
158
+ ## License and Legal Information
159
+
160
+ This repository contains open-source code licensed under the MIT License. A copy of the license can be found in the [license](./license) file.
161
+
162
+ **Please note:** The MIT License does not grant permission to use the trade names, trademarks, service marks, or product names of the project, except as required for reasonable and customary use in describing the origin of the work and reproducing the content of the NOTICE file.
163
+
164
+ ### Trademarks
165
+
166
+ This project is owned and maintained by Task Venture Capital GmbH. The names and logos associated with Task Venture Capital GmbH and any related products or services are trademarks of Task Venture Capital GmbH or third parties, and are not included within the scope of the MIT license granted herein.
167
+
168
+ Use of these trademarks must comply with Task Venture Capital GmbH's Trademark Guidelines or the guidelines of the respective third-party owners, and any usage must be approved in writing. Third-party trademarks used herein are the property of their respective owners and used only in a descriptive manner, e.g. for an implementation of an API or similar.
169
+
170
+ ### Company Information
171
+
172
+ Task Venture Capital GmbH<br>
173
+ Registered at District Court Bremen HRB 35230 HB, Germany
174
+
175
+ For any legal inquiries or further information, please contact us via email at hello@task.vc.
176
+
177
+ By using this repository, you acknowledge that you have read this section, agree to comply with its terms, and understand that the licensing of the code does not imply endorsement by Task Venture Capital GmbH of any derivative works.
package/readme.todo.md ADDED
@@ -0,0 +1,3 @@
1
+ # GitOps TODOs
2
+
3
+ - [ ] Webhook HMAC signature verification (X-Gitea-Signature / X-Gitlab-Token) — currently accepts all POSTs
@@ -0,0 +1,8 @@
1
+ /**
2
+ * autocreated commitinfo by @push.rocks/commitinfo
3
+ */
4
+ export const commitinfo = {
5
+ name: '@serve.zone/gitops',
6
+ version: '2.13.1',
7
+ description: 'GitOps management app for Gitea and GitLab - manage secrets, browse projects, view CI pipelines, and stream build logs'
8
+ }
@@ -0,0 +1,69 @@
1
+ import { logger } from '../logging.js';
2
+ import { unrefTimer } from '../timers.js';
3
+ import type { CacheDb } from './classes.cachedb.js';
4
+
5
+ // deno-lint-ignore no-explicit-any
6
+ type DocumentClass = { getInstances: (filter: any) => Promise<{ delete: () => Promise<void> }[]> };
7
+
8
+ const DEFAULT_INTERVAL_MS = 60 * 60 * 1000; // 1 hour
9
+
10
+ /**
11
+ * Periodically cleans up expired cached documents.
12
+ */
13
+ export class CacheCleaner {
14
+ private intervalId: ReturnType<typeof setInterval> | null = null;
15
+ private intervalMs: number;
16
+ private documentClasses: DocumentClass[] = [];
17
+ private cacheDb: CacheDb;
18
+
19
+ constructor(cacheDb: CacheDb, intervalMs = DEFAULT_INTERVAL_MS) {
20
+ this.cacheDb = cacheDb;
21
+ this.intervalMs = intervalMs;
22
+ }
23
+
24
+ /** Register a document class for cleanup */
25
+ registerClass(cls: DocumentClass): void {
26
+ this.documentClasses.push(cls);
27
+ }
28
+
29
+ start(): void {
30
+ if (this.intervalId !== null) return;
31
+ this.intervalId = setInterval(() => {
32
+ this.clean().catch((err) => {
33
+ logger.error(`CacheCleaner error: ${err}`);
34
+ });
35
+ }, this.intervalMs);
36
+ // Unref so the interval doesn't prevent process exit
37
+ unrefTimer(this.intervalId);
38
+ logger.debug(`CacheCleaner started (interval: ${this.intervalMs}ms)`);
39
+ }
40
+
41
+ stop(): void {
42
+ if (this.intervalId !== null) {
43
+ clearInterval(this.intervalId);
44
+ this.intervalId = null;
45
+ logger.debug('CacheCleaner stopped');
46
+ }
47
+ }
48
+
49
+ /** Run a single cleanup pass */
50
+ async clean(): Promise<number> {
51
+ const now = Date.now();
52
+ let totalDeleted = 0;
53
+ for (const cls of this.documentClasses) {
54
+ try {
55
+ const expired = await cls.getInstances({ expiresAt: { $lt: now } });
56
+ for (const doc of expired) {
57
+ await doc.delete();
58
+ totalDeleted++;
59
+ }
60
+ } catch (err) {
61
+ logger.error(`CacheCleaner: failed to clean class: ${err}`);
62
+ }
63
+ }
64
+ if (totalDeleted > 0) {
65
+ logger.debug(`CacheCleaner: deleted ${totalDeleted} expired document(s)`);
66
+ }
67
+ return totalDeleted;
68
+ }
69
+ }
@@ -0,0 +1,57 @@
1
+ import * as plugins from '../plugins.js';
2
+
3
+ /** TTL duration constants in milliseconds */
4
+ export const TTL = {
5
+ MINUTES_5: 5 * 60 * 1000,
6
+ HOURS_1: 60 * 60 * 1000,
7
+ HOURS_24: 24 * 60 * 60 * 1000,
8
+ DAYS_7: 7 * 24 * 60 * 60 * 1000,
9
+ DAYS_30: 30 * 24 * 60 * 60 * 1000,
10
+ DAYS_90: 90 * 24 * 60 * 60 * 1000,
11
+ } as const;
12
+
13
+ /**
14
+ * Abstract base class for cached documents with TTL support.
15
+ * Extend this class and add @Collection decorator pointing to your CacheDb.
16
+ */
17
+ export abstract class CachedDocument<
18
+ T extends CachedDocument<T>,
19
+ > extends plugins.smartdata.SmartDataDbDoc<T, T> {
20
+ @plugins.smartdata.svDb()
21
+ public createdAt: number = Date.now();
22
+
23
+ @plugins.smartdata.svDb()
24
+ public expiresAt: number = Date.now() + TTL.HOURS_1;
25
+
26
+ @plugins.smartdata.svDb()
27
+ public lastAccessedAt: number = Date.now();
28
+
29
+ constructor() {
30
+ super();
31
+ }
32
+
33
+ /** Set TTL in milliseconds from now */
34
+ setTTL(ms: number): void {
35
+ this.expiresAt = Date.now() + ms;
36
+ }
37
+
38
+ /** Set TTL in days from now */
39
+ setTTLDays(days: number): void {
40
+ this.setTTL(days * 24 * 60 * 60 * 1000);
41
+ }
42
+
43
+ /** Set TTL in hours from now */
44
+ setTTLHours(hours: number): void {
45
+ this.setTTL(hours * 60 * 60 * 1000);
46
+ }
47
+
48
+ /** Check if this document has expired */
49
+ isExpired(): boolean {
50
+ return Date.now() > this.expiresAt;
51
+ }
52
+
53
+ /** Update last accessed timestamp */
54
+ touch(): void {
55
+ this.lastAccessedAt = Date.now();
56
+ }
57
+ }
@@ -0,0 +1,72 @@
1
+ import * as plugins from '../plugins.js';
2
+ import { logger } from '../logging.js';
3
+
4
+ export interface ICacheDbOptions {
5
+ storagePath?: string;
6
+ dbName?: string;
7
+ debug?: boolean;
8
+ }
9
+
10
+ /**
11
+ * Singleton wrapper around LocalTsmDb + SmartdataDb.
12
+ * Provides a managed MongoDB-compatible cache database.
13
+ */
14
+ export class CacheDb {
15
+ private static instance: CacheDb | null = null;
16
+
17
+ private smartMongo: InstanceType<typeof plugins.smartmongo.SmartMongo> | null = null;
18
+ private smartdataDb: InstanceType<typeof plugins.smartdata.SmartdataDb> | null = null;
19
+ private options: Required<ICacheDbOptions>;
20
+
21
+ private constructor(options: ICacheDbOptions = {}) {
22
+ this.options = {
23
+ storagePath: options.storagePath ?? './.nogit/cachedb',
24
+ dbName: options.dbName ?? 'gitops_cache',
25
+ debug: options.debug ?? false,
26
+ };
27
+ }
28
+
29
+ static getInstance(options?: ICacheDbOptions): CacheDb {
30
+ if (!CacheDb.instance) {
31
+ CacheDb.instance = new CacheDb(options);
32
+ }
33
+ return CacheDb.instance;
34
+ }
35
+
36
+ static resetInstance(): void {
37
+ CacheDb.instance = null;
38
+ }
39
+
40
+ async start(): Promise<void> {
41
+ logger.info('Starting CacheDb...');
42
+ this.smartMongo = await plugins.smartmongo.SmartMongo.createAndStart();
43
+ const mongoDescriptor = await this.smartMongo.getMongoDescriptor();
44
+
45
+ this.smartdataDb = new plugins.smartdata.SmartdataDb({
46
+ mongoDbUrl: mongoDescriptor.mongoDbUrl,
47
+ mongoDbName: this.options.dbName,
48
+ });
49
+ await this.smartdataDb.init();
50
+ logger.success(`CacheDb started (db: ${this.options.dbName})`);
51
+ }
52
+
53
+ async stop(): Promise<void> {
54
+ logger.info('Stopping CacheDb...');
55
+ if (this.smartdataDb) {
56
+ await this.smartdataDb.close();
57
+ this.smartdataDb = null;
58
+ }
59
+ if (this.smartMongo) {
60
+ await this.smartMongo.stop();
61
+ this.smartMongo = null;
62
+ }
63
+ logger.success('CacheDb stopped');
64
+ }
65
+
66
+ getDb(): InstanceType<typeof plugins.smartdata.SmartdataDb> {
67
+ if (!this.smartdataDb) {
68
+ throw new Error('CacheDb not started. Call start() first.');
69
+ }
70
+ return this.smartdataDb;
71
+ }
72
+ }
@@ -0,0 +1,267 @@
1
+ import { logger } from '../logging.js';
2
+ import type { ConnectionManager } from '../classes/connectionmanager.js';
3
+ import { CachedSecret } from './documents/classes.cached.secret.js';
4
+ import { TTL } from './classes.cached.document.js';
5
+ import type { ISecret } from '../../ts_interfaces/data/secret.js';
6
+
7
+ export interface IScanResult {
8
+ connectionsScanned: number;
9
+ secretsFound: number;
10
+ errors: string[];
11
+ durationMs: number;
12
+ }
13
+
14
+ /**
15
+ * Centralized secrets scanning service. Fetches all secrets from all
16
+ * connections and upserts them into the CachedSecret collection.
17
+ */
18
+ export class SecretsScanService {
19
+ public lastScanTimestamp: number = 0;
20
+ public lastScanResult: IScanResult | null = null;
21
+ public isScanning: boolean = false;
22
+
23
+ private connectionManager: ConnectionManager;
24
+
25
+ constructor(connectionManager: ConnectionManager) {
26
+ this.connectionManager = connectionManager;
27
+ }
28
+
29
+ /**
30
+ * Upsert a single secret into the cache. If a doc with the same composite ID
31
+ * already exists, update it in place; otherwise insert a new one.
32
+ */
33
+ private async upsertSecret(secret: ISecret): Promise<void> {
34
+ const id = CachedSecret.buildId(secret.connectionId, secret.scope, secret.scopeId, secret.key);
35
+ const existing = await CachedSecret.getInstance({ id });
36
+ if (existing) {
37
+ existing.value = secret.value;
38
+ existing.protected = secret.protected;
39
+ existing.masked = secret.masked;
40
+ existing.environment = secret.environment;
41
+ existing.scopeName = secret.scopeName;
42
+ existing.setTTL(TTL.HOURS_24);
43
+ await existing.save();
44
+ } else {
45
+ const doc = CachedSecret.fromISecret(secret);
46
+ await doc.save();
47
+ }
48
+ }
49
+
50
+ /**
51
+ * Save an array of secrets to cache using upsert logic.
52
+ * Best-effort: individual failures are silently ignored.
53
+ */
54
+ async saveSecrets(secrets: ISecret[]): Promise<void> {
55
+ for (const secret of secrets) {
56
+ try {
57
+ await this.upsertSecret(secret);
58
+ } catch {
59
+ // Best-effort caching
60
+ }
61
+ }
62
+ }
63
+
64
+ /**
65
+ * Full scan: iterate all connections, fetch all projects+groups,
66
+ * fetch all secrets per entity, upsert CachedSecret docs.
67
+ */
68
+ async fullScan(): Promise<IScanResult> {
69
+ if (this.isScanning) {
70
+ return {
71
+ connectionsScanned: 0,
72
+ secretsFound: 0,
73
+ errors: ['Scan already in progress'],
74
+ durationMs: 0,
75
+ };
76
+ }
77
+
78
+ this.isScanning = true;
79
+ const startTime = Date.now();
80
+ const errors: string[] = [];
81
+ let totalSecrets = 0;
82
+ let connectionsScanned = 0;
83
+
84
+ try {
85
+ const connections = this.connectionManager.getConnections();
86
+ for (const conn of connections) {
87
+ if (conn.status === 'paused') continue;
88
+ try {
89
+ const provider = this.connectionManager.getProvider(conn.id);
90
+ connectionsScanned++;
91
+
92
+ // Scan project secrets
93
+ try {
94
+ const projects = await provider.getProjects();
95
+ for (let i = 0; i < projects.length; i += 5) {
96
+ const batch = projects.slice(i, i + 5);
97
+ const results = await Promise.allSettled(
98
+ batch.map(async (p) => {
99
+ const secrets = await provider.getProjectSecrets(p.id);
100
+ return secrets.map((s) => ({
101
+ ...s,
102
+ scope: 'project' as const,
103
+ scopeId: p.id,
104
+ scopeName: p.fullPath || p.name,
105
+ connectionId: conn.id,
106
+ }));
107
+ }),
108
+ );
109
+ for (const result of results) {
110
+ if (result.status === 'fulfilled') {
111
+ for (const secret of result.value) {
112
+ try {
113
+ await this.upsertSecret(secret);
114
+ totalSecrets++;
115
+ } catch (err) {
116
+ errors.push(`Save secret ${secret.key}: ${err}`);
117
+ }
118
+ }
119
+ } else {
120
+ errors.push(`Fetch project secrets: ${result.reason}`);
121
+ }
122
+ }
123
+ }
124
+ } catch (err) {
125
+ errors.push(`Fetch projects for ${conn.id}: ${err}`);
126
+ }
127
+
128
+ // Scan group secrets
129
+ try {
130
+ const groups = await provider.getGroups();
131
+ for (let i = 0; i < groups.length; i += 5) {
132
+ const batch = groups.slice(i, i + 5);
133
+ const results = await Promise.allSettled(
134
+ batch.map(async (g) => {
135
+ const secrets = await provider.getGroupSecrets(g.id);
136
+ return secrets.map((s) => ({
137
+ ...s,
138
+ scope: 'group' as const,
139
+ scopeId: g.id,
140
+ scopeName: g.fullPath || g.name,
141
+ connectionId: conn.id,
142
+ }));
143
+ }),
144
+ );
145
+ for (const result of results) {
146
+ if (result.status === 'fulfilled') {
147
+ for (const secret of result.value) {
148
+ try {
149
+ await this.upsertSecret(secret);
150
+ totalSecrets++;
151
+ } catch (err) {
152
+ errors.push(`Save secret ${secret.key}: ${err}`);
153
+ }
154
+ }
155
+ } else {
156
+ errors.push(`Fetch group secrets: ${result.reason}`);
157
+ }
158
+ }
159
+ }
160
+ } catch (err) {
161
+ errors.push(`Fetch groups for ${conn.id}: ${err}`);
162
+ }
163
+ } catch (err) {
164
+ errors.push(`Connection ${conn.id}: ${err}`);
165
+ }
166
+ }
167
+ } finally {
168
+ this.isScanning = false;
169
+ }
170
+
171
+ const result: IScanResult = {
172
+ connectionsScanned,
173
+ secretsFound: totalSecrets,
174
+ errors,
175
+ durationMs: Date.now() - startTime,
176
+ };
177
+
178
+ this.lastScanTimestamp = Date.now();
179
+ this.lastScanResult = result;
180
+
181
+ logger.info(
182
+ `Secrets scan complete: ${totalSecrets} secrets from ${connectionsScanned} connections in ${result.durationMs}ms` +
183
+ (errors.length > 0 ? ` (${errors.length} errors)` : ''),
184
+ );
185
+
186
+ return result;
187
+ }
188
+
189
+ /**
190
+ * Scan a single entity: delete existing cached secrets for that entity,
191
+ * fetch fresh from provider, and save to cache.
192
+ */
193
+ async scanEntity(
194
+ connectionId: string,
195
+ scope: 'project' | 'group',
196
+ scopeId: string,
197
+ scopeName?: string,
198
+ ): Promise<void> {
199
+ try {
200
+ // Delete existing cached secrets for this entity
201
+ const existing = await CachedSecret.getInstances({
202
+ connectionId,
203
+ scope,
204
+ scopeId,
205
+ });
206
+ for (const doc of existing) {
207
+ await doc.delete();
208
+ }
209
+
210
+ // Fetch fresh from provider
211
+ const provider = this.connectionManager.getProvider(connectionId);
212
+ const secrets = scope === 'project'
213
+ ? await provider.getProjectSecrets(scopeId)
214
+ : await provider.getGroupSecrets(scopeId);
215
+
216
+ // Save to cache
217
+ for (const s of secrets) {
218
+ const doc = CachedSecret.fromISecret({
219
+ ...s,
220
+ scope,
221
+ scopeId,
222
+ scopeName: scopeName || s.scopeName || '',
223
+ connectionId,
224
+ });
225
+ await doc.save();
226
+ }
227
+ } catch (err) {
228
+ logger.error(`scanEntity failed for ${connectionId}/${scope}/${scopeId}: ${err}`);
229
+ }
230
+ }
231
+
232
+ /**
233
+ * Get cached secrets matching the filter criteria.
234
+ */
235
+ async getCachedSecrets(filter: {
236
+ connectionId: string;
237
+ scope: 'project' | 'group';
238
+ scopeId?: string;
239
+ }): Promise<ISecret[]> {
240
+ // deno-lint-ignore no-explicit-any
241
+ const query: any = {
242
+ connectionId: filter.connectionId,
243
+ scope: filter.scope,
244
+ };
245
+ if (filter.scopeId) {
246
+ query.scopeId = filter.scopeId;
247
+ }
248
+ const docs = await CachedSecret.getInstances(query);
249
+ // Filter out expired docs
250
+ const now = Date.now();
251
+ return docs
252
+ .filter((d) => d.expiresAt > now)
253
+ .map((d) => d.toISecret());
254
+ }
255
+
256
+ /**
257
+ * Check if non-expired cached data exists for the given connection+scope.
258
+ */
259
+ async hasCachedData(connectionId: string, scope: 'project' | 'group'): Promise<boolean> {
260
+ const docs = await CachedSecret.getInstances({
261
+ connectionId,
262
+ scope,
263
+ expiresAt: { $gt: Date.now() },
264
+ });
265
+ return docs.length > 0;
266
+ }
267
+ }
@@ -0,0 +1,32 @@
1
+ import * as plugins from '../../plugins.js';
2
+ import { CacheDb } from '../classes.cachedb.js';
3
+ import { CachedDocument, TTL } from '../classes.cached.document.js';
4
+
5
+ /**
6
+ * Cached project data from git providers. TTL: 5 minutes.
7
+ */
8
+ @plugins.smartdata.Collection(() => CacheDb.getInstance().getDb())
9
+ export class CachedProject extends CachedDocument<CachedProject> {
10
+ @plugins.smartdata.unI()
11
+ public id: string = '';
12
+
13
+ @plugins.smartdata.svDb()
14
+ public connectionId: string = '';
15
+
16
+ @plugins.smartdata.svDb()
17
+ public projectName: string = '';
18
+
19
+ @plugins.smartdata.svDb()
20
+ public projectUrl: string = '';
21
+
22
+ @plugins.smartdata.svDb()
23
+ public description: string = '';
24
+
25
+ @plugins.smartdata.svDb()
26
+ public defaultBranch: string = '';
27
+
28
+ constructor() {
29
+ super();
30
+ this.setTTL(TTL.MINUTES_5);
31
+ }
32
+ }