vector_amp 0.3.0 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7147209a61689dee14e2d37581f33cad58fb5291543edb086fcafdf841e71c0a
-  data.tar.gz: c399a9373b4820d6c141362b29e2b81bc34de71a02ecb0d69c7780febdbd3674
+  metadata.gz: 8275404bc4db2f1270091d20315634d273fa140417503bdfea605c9d3baf0c5b
+  data.tar.gz: 426ae6dd92df94fd947d92ace32cbae23af4b0b0b69cf6a7eb2e9daf5e380257
 SHA512:
-  metadata.gz: 6b2f983acc42db9c33801bcbe726537975af6d6ec9eab12a542a860ac5836fa417e07bb35ddfd6e4f7a55f0b6311c4ee5793d192427e5182c5fff11aae9849c4
-  data.tar.gz: 2d720c1922917177c98cf7ddab94b9f2f20ef0654e31f673d47f4283459cca5c9808851875bf2a28a40ff9f6d63a7634bd852b0a74539453c6e7dff9d4dd58b1
+  metadata.gz: 585008c3ef2c83a95d8e55956ac4e11a6b006f05e3aa8d3af3d5932967edf53ce3f47af88612dd9e56e443eea327b4de30cdd6b82e5732d0d93e15a67edfc7ea
+  data.tar.gz: ab6d8685a761322203f41278049106953c0c3a12a3e3b12cd844cc2c7c3d89745027d3df9c1745ac41b656708a84b742eb199156dcf6e8e038c795ba7a5c5a5f

data/README.md

@@ -197,7 +197,8 @@ client.ingestion.list_sources(limit: 50, offset: 0)
 client.ingestion.get_source("source-uuid")
 
 # Typed builders cover the supported source_type values:
-# "web", "s3", "gcs", "gdrive", "file_upload", "jira", and "confluence".
+# "web", "s3", "gcs", "gdrive", "file_upload", "jira", "confluence",
+# "github", and "gitlab".
 source = client.sources.create_web(
   start_urls: ["https://docs.example.com"],
   max_depth: 1,
@@ -229,6 +230,20 @@ gdrive_source = client.sources.create_google_drive(
   folder_ids: ["google-drive-folder-id"]
 )
 
+# GitHub reads through the VectorAmp GitHub App, so the SDK never takes a token.
+# Install the app from the Sources page and pass the installation id it reports.
+github_source = client.sources.create_github(
+  installation_id: 42,
+  repositories: ["acme/api", "acme/web"]
+)
+
+gitlab_source = client.sources.create_gitlab(
+  projects: ["mygroup/myproject"], # and/or groups: ["mygroup"]
+  auth_mode: "token",              # defaults to "oauth"
+  access_token: ENV["GITLAB_TOKEN"],
+  gitlab_url: "https://gitlab.com" # override for self-managed
+)
+
 file_source = client.sources.create_file_upload
 
 # You can still pass name: when you want control. If omitted, the SDK chooses
@@ -406,7 +421,8 @@ Dataset-object helpers: `search`, `insert`, `add_texts`, `embed`, `delete`, `sta
 
 `create(source)`, `create_web(start_urls:)`, `create_s3(bucket:)`, `create_gcs(bucket:)`,
 `create_google_drive(folder_ids:|file_ids:)`, `create_jira(cloud_id:)`,
-`create_confluence(cloud_id:|base_url:)`, `create_file_upload`, `list_sources`, `get_source(id)`.
+`create_confluence(cloud_id:|base_url:)`, `create_github(installation_id:, repositories:)`,
+`create_gitlab(groups:|projects:)`, `create_file_upload`, `list_sources`, `get_source(id)`.
 Jobs: `start_job(source_id:, dataset_id:)`, `list_jobs`, `get_job(id)`, `retry_job(id)`,
 `ingest_files(dataset_id:, paths:)`.
 

data/lib/vector_amp/ingestion.rb

@@ -224,6 +224,102 @@ module VectorAmp
       ))
     end
 
+    # Create a GitHub source backed by the VectorAmp GitHub App.
+    #
+    # Install the VectorAmp GitHub App from the Sources page in the app first and
+    # pass the resulting installation id here; the SDK never handles a GitHub token.
+    # @param installation_id [Integer] required GitHub App installation id.
+    # @param repositories [String, Array<String>] required `owner/repo` full names.
+    # @param name [String, nil] defaults to `github-<owner>-<repo>` from the first repository.
+    # @param ref_mode [String, nil] `active` (server default), `default`, or `explicit`.
+    # @param refs [String, Array<String>, nil] explicit branch names for `ref_mode: "explicit"`.
+    # @param excluded_refs [String, Array<String>, nil] branch names to skip.
+    # @param active_branch_days [Integer, nil] activity window in days (1-90); server default 7.
+    # @param include_pull_requests [Boolean, nil] ingest pull requests; server default true.
+    # @param include_review_threads [Boolean, nil] ingest review discussions; server default true.
+    # @param include_direct_commits [Boolean, nil] ingest commits outside a pull request; server default true.
+    # @param include_globs [String, Array<String>, nil] path globs to include; server default `**/*`.
+    # @param exclude_globs [String, Array<String>, nil] path globs to skip.
+    # @param max_file_size_bytes [Integer, nil] per-file ceiling; server default 1_000_000.
+    # @param description [String, nil] optional description.
+    # @param metadata [Hash, nil] optional metadata.
+    # @param config [Hash] additional GitHub-source config forwarded to the API.
+    # @return [Hash] created source response.
+    def create_github(installation_id:, repositories:, name: nil, ref_mode: nil, refs: nil, excluded_refs: nil,
+                      active_branch_days: nil, include_pull_requests: nil, include_review_threads: nil,
+                      include_direct_commits: nil, include_globs: nil, exclude_globs: nil,
+                      max_file_size_bytes: nil, description: nil, metadata: nil, **config)
+      create_source(GitHubSource.new(
+        installation_id: installation_id,
+        repositories: repositories,
+        name: name,
+        ref_mode: ref_mode,
+        refs: refs,
+        excluded_refs: excluded_refs,
+        active_branch_days: active_branch_days,
+        include_pull_requests: include_pull_requests,
+        include_review_threads: include_review_threads,
+        include_direct_commits: include_direct_commits,
+        include_globs: include_globs,
+        exclude_globs: exclude_globs,
+        max_file_size_bytes: max_file_size_bytes,
+        description: description,
+        metadata: metadata,
+        **config
+      ))
+    end
+
+    # Create a GitLab source for gitlab.com or a self-managed instance.
+    # @param groups [String, Array<String>, nil] group paths; required unless projects is given.
+    # @param projects [String, Array<String>, nil] project paths with namespace; required unless groups is given.
+    # @param name [String, nil] defaults to `gitlab-<path>` from the first project or group.
+    # @param auth_mode [String] `oauth` (default) or `token`.
+    # @param gitlab_url [String] instance base URL; defaults to `https://gitlab.com`.
+    # @param access_token [String, nil] personal or group access token for `auth_mode: "token"`.
+    # @param connection_id [String, nil] optional stored OAuth connection id used instead of a token.
+    # @param ref_mode [String, nil] `active` (server default), `default`, or `explicit`.
+    # @param refs [String, Array<String>, nil] explicit branch names for `ref_mode: "explicit"`.
+    # @param excluded_refs [String, Array<String>, nil] branch names to skip.
+    # @param active_branch_days [Integer, nil] activity window in days (1-90); server default 7.
+    # @param include_merge_requests [Boolean, nil] ingest merge requests; server default true.
+    # @param include_review_threads [Boolean, nil] ingest review discussions; server default true.
+    # @param include_direct_commits [Boolean, nil] ingest commits outside a merge request; server default true.
+    # @param include_globs [String, Array<String>, nil] path globs to include; server default `**/*`.
+    # @param exclude_globs [String, Array<String>, nil] path globs to skip.
+    # @param max_file_size_bytes [Integer, nil] per-file ceiling; server default 1_000_000.
+    # @param description [String, nil] optional description.
+    # @param metadata [Hash, nil] optional metadata.
+    # @param config [Hash] additional GitLab-source config forwarded to the API.
+    # @return [Hash] created source response.
+    def create_gitlab(groups: nil, projects: nil, name: nil, auth_mode: "oauth", gitlab_url: "https://gitlab.com",
+                      access_token: nil, connection_id: nil, ref_mode: nil, refs: nil, excluded_refs: nil,
+                      active_branch_days: nil, include_merge_requests: nil, include_review_threads: nil,
+                      include_direct_commits: nil, include_globs: nil, exclude_globs: nil,
+                      max_file_size_bytes: nil, description: nil, metadata: nil, **config)
+      create_source(GitLabSource.new(
+        groups: groups,
+        projects: projects,
+        name: name,
+        auth_mode: auth_mode,
+        gitlab_url: gitlab_url,
+        access_token: access_token,
+        connection_id: connection_id,
+        ref_mode: ref_mode,
+        refs: refs,
+        excluded_refs: excluded_refs,
+        active_branch_days: active_branch_days,
+        include_merge_requests: include_merge_requests,
+        include_review_threads: include_review_threads,
+        include_direct_commits: include_direct_commits,
+        include_globs: include_globs,
+        exclude_globs: exclude_globs,
+        max_file_size_bytes: max_file_size_bytes,
+        description: description,
+        metadata: metadata,
+        **config
+      ))
+    end
+
     # Create a file-upload source.
     # @param name [String, nil] defaults to timestamped `ruby-sdk-file-upload-YYYYmmddHHMMSS`.
     # @param description [String, nil] optional description.
@@ -386,6 +482,8 @@ module VectorAmp
       when "jira" then SourceNames.jira(project_keys: config[:project_keys] || config["project_keys"], cloud_id: config[:cloud_id] || config["cloud_id"])
       when "confluence" then SourceNames.confluence(spaces: config[:spaces] || config["spaces"], cloud_id: config[:cloud_id] || config["cloud_id"], base_url: config[:base_url] || config["base_url"])
       when "gdrive" then SourceNames.google_drive(folder_ids: config[:folder_ids] || config["folder_ids"], file_ids: config[:file_ids] || config["file_ids"])
+      when "github" then SourceNames.github(repositories: config[:repositories] || config["repositories"])
+      when "gitlab" then SourceNames.gitlab(projects: config[:projects] || config["projects"], groups: config[:groups] || config["groups"])
       end
     end
 

data/lib/vector_amp/source.rb

@@ -10,7 +10,7 @@ module VectorAmp
   # create a source or to `dataset.ingest_source(source)` once they include an id
   # returned by the API.
   class Source
-    SUPPORTED_SOURCE_TYPES = %w[s3 web gcs gdrive file_upload jira confluence].freeze
+    SUPPORTED_SOURCE_TYPES = %w[s3 web gcs gdrive file_upload jira confluence github gitlab].freeze
 
     # @return [String, nil] API source id when returned by the API.
     # @return [String] source type (`s3`, `web`, `gdrive`, or `file_upload`).
@@ -164,6 +164,28 @@ module VectorAmp
       nil
     end
 
+    # @param repositories [String, Array<String>, nil] `owner/repo` full names.
+    # @return [String] `github-<owner>-<repo>` from the first repository, or `github-source`.
+    def github(repositories: nil)
+      repository = Array(repositories).first
+      repository ? "github-#{slugify(repository)}" : "github-source"
+    end
+
+    # @param projects [String, Array<String>, nil] project paths with namespace.
+    # @param groups [String, Array<String>, nil] group paths.
+    # @return [String] `gitlab-<path>` from the first project or group, or `gitlab-source`.
+    def gitlab(projects: nil, groups: nil)
+      path = Array(projects).first || Array(groups).first
+      path ? "gitlab-#{slugify(path)}" : "gitlab-source"
+    end
+
+    # Lowercase a repository/group path into a dash-separated name fragment.
+    # @param value [String] path such as `owner/repo` or `group/subgroup/project`.
+    # @return [String]
+    def slugify(value)
+      value.to_s.downcase.gsub(/[^a-z0-9]+/, "-").gsub(/\A-+|-+\z/, "")
+    end
+
     # @param folder_ids [String, Array<String>, nil] folder ids.
     # @param file_ids [String, Array<String>, nil] file ids.
     # @return [String] `google-drive-<first id>` or `google-drive-source`.
@@ -360,6 +382,133 @@ module VectorAmp
     end
   end
 
+  # GitHub ingestion source backed by the VectorAmp GitHub App.
+  #
+  # Repositories, active branches, pull requests, and review discussions are read
+  # through a GitHub App installation, so no token is passed to the SDK. Install
+  # the VectorAmp GitHub App from the Sources page in the app first; the
+  # installation id shown there is what this class needs.
+  class GitHubSource < Source
+    # @param installation_id [Integer] required GitHub App installation id; must be positive.
+    # @param repositories [String, Array<String>] required `owner/repo` full names.
+    # @param name [String, nil] defaults to `github-<owner>-<repo>` from the first repository.
+    # @param ref_mode [String, nil] `active` (server default), `default`, or `explicit`.
+    # @param refs [String, Array<String>, nil] explicit branch names for `ref_mode: "explicit"`.
+    # @param excluded_refs [String, Array<String>, nil] branch names to skip.
+    # @param active_branch_days [Integer, nil] activity window in days (1-90); server default 7.
+    # @param include_pull_requests [Boolean, nil] ingest pull requests; server default true.
+    # @param include_review_threads [Boolean, nil] ingest review discussions; server default true.
+    # @param include_direct_commits [Boolean, nil] ingest commits outside a pull request; server default true.
+    # @param include_globs [String, Array<String>, nil] path globs to include; server default `**/*`.
+    # @param exclude_globs [String, Array<String>, nil] path globs to skip.
+    # @param max_file_size_bytes [Integer, nil] per-file ceiling (1-25_000_000); server default 1_000_000.
+    # @param description [String, nil] optional description.
+    # @param metadata [Hash, nil] optional metadata.
+    # @param id [String, nil] optional API source id.
+    # @param config [Hash] additional GitHub-source config forwarded to the API.
+    # @return [GitHubSource]
+    def initialize(installation_id:, repositories:, name: nil, ref_mode: nil, refs: nil, excluded_refs: nil,
+                   active_branch_days: nil, include_pull_requests: nil, include_review_threads: nil,
+                   include_direct_commits: nil, include_globs: nil, exclude_globs: nil, max_file_size_bytes: nil,
+                   description: nil, metadata: nil, id: nil, **config)
+      unless installation_id.is_a?(Integer) && installation_id.positive?
+        raise ArgumentError, "installation_id must be a positive Integer"
+      end
+
+      repository_list = Array(repositories)
+      raise ArgumentError, "repositories must not be empty" if repository_list.empty?
+
+      super(
+        id: id,
+        source_type: "github",
+        name: name || SourceNames.github(repositories: repository_list),
+        description: description,
+        metadata: metadata,
+        config: Utils.compact_hash(config.merge(
+          installation_id: installation_id,
+          repositories: repository_list,
+          ref_mode: ref_mode,
+          refs: refs.nil? ? nil : Array(refs),
+          excluded_refs: excluded_refs.nil? ? nil : Array(excluded_refs),
+          active_branch_days: active_branch_days,
+          include_pull_requests: include_pull_requests,
+          include_review_threads: include_review_threads,
+          include_direct_commits: include_direct_commits,
+          include_globs: include_globs.nil? ? nil : Array(include_globs),
+          exclude_globs: exclude_globs.nil? ? nil : Array(exclude_globs),
+          max_file_size_bytes: max_file_size_bytes
+        ))
+      )
+    end
+  end
+
+  # GitLab ingestion source for gitlab.com or a self-managed instance.
+  #
+  # Authenticates with an access token (`auth_mode: "token"` plus `access_token`)
+  # or a stored OAuth connection (`connection_id`). At least one group or project
+  # is required.
+  class GitLabSource < Source
+    # @param groups [String, Array<String>, nil] group paths; required unless projects is given.
+    # @param projects [String, Array<String>, nil] project paths with namespace; required unless groups is given.
+    # @param name [String, nil] defaults to `gitlab-<path>` from the first project or group.
+    # @param auth_mode [String] `oauth` (default) or `token`.
+    # @param gitlab_url [String] instance base URL; defaults to `https://gitlab.com`.
+    # @param access_token [String, nil] personal or group access token for `auth_mode: "token"`.
+    # @param connection_id [String, nil] optional stored OAuth connection id used instead of a token.
+    # @param ref_mode [String, nil] `active` (server default), `default`, or `explicit`.
+    # @param refs [String, Array<String>, nil] explicit branch names for `ref_mode: "explicit"`.
+    # @param excluded_refs [String, Array<String>, nil] branch names to skip.
+    # @param active_branch_days [Integer, nil] activity window in days (1-90); server default 7.
+    # @param include_merge_requests [Boolean, nil] ingest merge requests; server default true.
+    # @param include_review_threads [Boolean, nil] ingest review discussions; server default true.
+    # @param include_direct_commits [Boolean, nil] ingest commits outside a merge request; server default true.
+    # @param include_globs [String, Array<String>, nil] path globs to include; server default `**/*`.
+    # @param exclude_globs [String, Array<String>, nil] path globs to skip.
+    # @param max_file_size_bytes [Integer, nil] per-file ceiling (1-25_000_000); server default 1_000_000.
+    # @param description [String, nil] optional description.
+    # @param metadata [Hash, nil] optional metadata.
+    # @param id [String, nil] optional API source id.
+    # @param config [Hash] additional GitLab-source config forwarded to the API.
+    # @return [GitLabSource]
+    def initialize(groups: nil, projects: nil, name: nil, auth_mode: "oauth", gitlab_url: "https://gitlab.com",
+                   access_token: nil, connection_id: nil, ref_mode: nil, refs: nil, excluded_refs: nil,
+                   active_branch_days: nil, include_merge_requests: nil, include_review_threads: nil,
+                   include_direct_commits: nil, include_globs: nil, exclude_globs: nil, max_file_size_bytes: nil,
+                   description: nil, metadata: nil, id: nil, **config)
+      group_list = Array(groups)
+      project_list = Array(projects)
+      if group_list.empty? && project_list.empty?
+        raise ArgumentError, "groups or projects is required"
+      end
+
+      super(
+        id: id,
+        source_type: "gitlab",
+        name: name || SourceNames.gitlab(projects: project_list, groups: group_list),
+        description: description,
+        metadata: metadata,
+        config: Utils.compact_hash(config.merge(
+          auth_mode: auth_mode,
+          gitlab_url: gitlab_url,
+          groups: group_list.empty? ? nil : group_list,
+          projects: project_list.empty? ? nil : project_list,
+          access_token: access_token,
+          connection_id: connection_id,
+          ref_mode: ref_mode,
+          refs: refs.nil? ? nil : Array(refs),
+          excluded_refs: excluded_refs.nil? ? nil : Array(excluded_refs),
+          active_branch_days: active_branch_days,
+          include_merge_requests: include_merge_requests,
+          include_review_threads: include_review_threads,
+          include_direct_commits: include_direct_commits,
+          include_globs: include_globs.nil? ? nil : Array(include_globs),
+          exclude_globs: exclude_globs.nil? ? nil : Array(exclude_globs),
+          max_file_size_bytes: max_file_size_bytes
+        ))
+      )
+    end
+  end
+
   # Escape hatch for API-compatible source types/configs not yet modeled by the SDK.
   class GenericSource < Source
   end

data/lib/vector_amp/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module VectorAmp
-  VERSION = "0.3.0"
+  VERSION = "0.4.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: vector_amp
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.4.0
 platform: ruby
 authors:
 - VectorAmp
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2026-07-20 00:00:00.000000000 Z
+date: 2026-08-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: minitest