logster 2.5.1 → 2.6.0

data/client-app/package.json

@@ -18,40 +18,43 @@
     "test": "ember test"
   },
   "devDependencies": {
-    "@ember/optional-features": "^0.6.3",
-    "broccoli-asset-rev": "^2.7.0",
-    "ember-ajax": "^5.0.0",
-    "ember-cli": "^3.8.3",
+    "@ember/optional-features": "^1.1.0",
+    "@glimmer/component": "^1.0.0",
+    "babel-eslint": "^10.0.3",
+    "broccoli-asset-rev": "^3.0.0",
+    "ember-auto-import": "^1.5.3",
+    "ember-cli": "^3.15.0",
     "ember-cli-app-version": "^3.2.0",
-    "ember-cli-babel": "^7.12.0",
+    "ember-cli-babel": "^7.13.0",
     "ember-cli-dependency-checker": "^3.2.0",
-    "ember-cli-eslint": "^4.2.3",
-    "ember-cli-htmlbars": "^3.1.0",
-    "ember-cli-htmlbars-inline-precompile": "^1.0.3",
-    "ember-cli-inject-live-reload": "^1.8.2",
+    "ember-cli-eslint": "^5.1.0",
+    "ember-cli-htmlbars": "^4.2.0",
+    "ember-cli-inject-live-reload": "^2.0.1",
     "ember-cli-sri": "^2.1.1",
     "ember-cli-template-lint": "^1.0.0-beta.3",
-    "ember-cli-uglify": "^2.1.0",
-    "ember-data": "~3.8.0",
-    "ember-export-application-global": "^2.0.0",
+    "ember-cli-uglify": "^3.0.0",
+    "ember-data": "~3.15.0",
+    "ember-export-application-global": "^2.0.1",
+    "ember-fetch": "^7.0.0",
     "ember-font-awesome": "^4.0.0-rc.4",
-    "ember-load-initializers": "^1.1.0",
+    "ember-load-initializers": "^2.1.1",
     "ember-maybe-import-regenerator": "^0.1.6",
-    "ember-qunit": "^3.4.1",
-    "ember-resolver": "^5.3.0",
-    "ember-source": "^3.8.3",
-    "ember-welcome-page": "^3.2.0",
-    "eslint-plugin-ember": "^5.4.0",
-    "jquery": "3.4.1",
+    "ember-qunit": "^4.6.0",
+    "ember-resolver": "^7.0.0",
+    "ember-source": "^3.15.0",
+    "eslint-plugin-ember": "^7.7.1",
+    "eslint-plugin-node": "^10.0.0",
     "loader.js": "^4.7.0",
     "prettier": "^1.19.1",
-    "qunit-dom": "^0.8.5"
+    "qunit-dom": "^0.9.2"
   },
   "engines": {
-    "node": "6.* || 8.* || >= 10.*"
+    "node": "8.* || >= 10.*"
+  },
+  "ember": {
+    "edition": "octane"
   },
   "dependencies": {
-    "lodash": "^4.17.15",
     "moment": "~2.22.2"
   }
 }

data/client-app/preload-json-manager.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+# This script takes care of updating the content of the preloaded json in app/index.html and tests/index.html
+# All you need to do is update the ruby hash of the corresponding file you want to update and run the script
+
+require 'bundler/inline'
+require 'json'
+require 'cgi'
+
+gemfile do
+  source 'https://rubygems.org'
+  gem 'nokogiri'
+end
+
+tests_index_html = {
+  env_expandable_keys: [],
+  gems_dir: "/var/www/discourse/vendor/bundle/ruby/2.6.0/gems/",
+  backtrace_links_enabled: true,
+  gems_data: [
+    {
+      name: "activerecord",
+      url: "https://github.com/rails/rails/tree/v6.0.1/activerecord"
+    }
+  ],
+  directories: [
+    {
+      path: "/var/www/discourse",
+      url: "https://github.com/discourse/discourse",
+      main_app: true
+    },
+    {
+      path: "/var/www/discourse/plugins/discourse-prometheus",
+      url: "https://github.com/discourse/discourse-prometheus"
+    }
+  ],
+  application_version: "ce512452b512b909c38e9c63f2a0e1f8c17a2399"
+}
+
+app_index_html = {
+  env_expandable_keys: [],
+  patterns_enabled: true,
+  gems_dir: "/home/sam/.rbenv/versions/2.1.2.discourse/lib/ruby/gems/2.1.0/gems/",
+  backtrace_links_enabled: true,
+  gems_data: [],
+  directories: [
+    {
+      path: "/home/sam/Source/discourse",
+      url: "https://github.com/discourse/discourse",
+      main_app: true
+    }
+  ],
+  application_version: "b329e23f8511b7248c0e4aee370a9f8a249e1b84"
+}
+
+types = { app: app_index_html, tests: tests_index_html }
+
+%i{app tests}.each do |type|
+  content = File.read("#{type}/index.html")
+  json = CGI.escapeHTML(JSON.generate(types[type]))
+  content.sub!(/data-preloaded=".*">$/, "data-preloaded=\"#{json}\">")
+  File.write("#{type}/index.html", content)
+end

data/client-app/testem.js

@@ -13,7 +13,6 @@ module.exports = {
         // --no-sandbox is needed when running Chrome inside a container
         process.env.CI ? '--no-sandbox' : null,
         '--headless',
-        '--disable-gpu',
         '--disable-dev-shm-usage',
         '--disable-software-rasterizer',
         '--mute-audio',

data/client-app/tests/index.html

@@ -6,7 +6,7 @@
     <title>ClientApp Tests</title>
     <meta name="description" content="">
     <meta name="viewport" content="width=device-width, initial-scale=1">
-    <meta id="preloaded-data" data-root-path="/logs" data-preloaded="{&quot;env_expandable_keys&quot;:[]}">
+    <meta id="preloaded-data" data-root-path="/logs" data-preloaded="{&quot;env_expandable_keys&quot;:[],&quot;gems_dir&quot;:&quot;/var/www/discourse/vendor/bundle/ruby/2.6.0/gems/&quot;,&quot;backtrace_links_enabled&quot;:true,&quot;gems_data&quot;:[{&quot;name&quot;:&quot;activerecord&quot;,&quot;url&quot;:&quot;https://github.com/rails/rails/tree/v6.0.1/activerecord&quot;}],&quot;directories&quot;:[{&quot;path&quot;:&quot;/var/www/discourse&quot;,&quot;url&quot;:&quot;https://github.com/discourse/discourse&quot;,&quot;main_app&quot;:true},{&quot;path&quot;:&quot;/var/www/discourse/plugins/discourse-prometheus&quot;,&quot;url&quot;:&quot;https://github.com/discourse/discourse-prometheus&quot;}],&quot;application_version&quot;:&quot;ce512452b512b909c38e9c63f2a0e1f8c17a2399&quot;}">
 
     {{content-for "head"}}
     {{content-for "test-head"}}

data/client-app/tests/integration/components/back-trace-test.js

@@ -0,0 +1,109 @@
+import { module, test } from "qunit";
+import { setupRenderingTest } from "ember-qunit";
+import { render, find, findAll } from "@ember/test-helpers";
+import hbs from "htmlbars-inline-precompile";
+import { uninitialize, mutatePreload } from "client-app/lib/preload";
+
+module("Integration | Component | back-trace", function(hooks) {
+  setupRenderingTest(hooks);
+
+  hooks.beforeEach(function() {
+    uninitialize();
+  });
+
+  hooks.afterEach(function() {
+    uninitialize();
+  });
+
+  test("backtrace lines display and work correctly", async function(assert) {
+    const backtrace = `/var/www/discourse/vendor/bundle/ruby/2.6.0/gems/activerecord-6.0.1/lib/active_record/relation/finder_methods.rb:317:in \`exists?'
+/var/www/discourse/lib/permalink_constraint.rb:6:in \`matches?'
+/var/www/discourse/plugins/discourse-prometheus/lib/middleware/metrics.rb:17:in \`call'`;
+    this.set("backtrace", backtrace);
+    await render(hbs`{{back-trace backtrace=backtrace}}`);
+
+    const [gem, app, plugin] = findAll("a");
+    assert.equal(
+      gem.href,
+      "https://github.com/rails/rails/tree/v6.0.1/activerecord/lib/active_record/relation/finder_methods.rb#L317"
+    );
+
+    assert.equal(
+      app.href,
+      "https://github.com/discourse/discourse/blob/ce512452b512b909c38e9c63f2a0e1f8c17a2399/lib/permalink_constraint.rb#L6"
+    );
+
+    assert.equal(
+      plugin.href,
+      "https://github.com/discourse/discourse-prometheus/blob/master/lib/middleware/metrics.rb#L17"
+    );
+
+    let gemLine = find("div.backtrace-line");
+    assert.equal(
+      gemLine.textContent,
+      "activerecord-6.0.1/lib/active_record/relation/finder_methods.rb:317:in `exists?'",
+      "gem lines are truncated"
+    );
+  });
+
+  test("non-ruby backtraces don't break things", async function(assert) {
+    this.set(
+      "backtrace",
+      `m/<@https://discourse-cdn.com/assets/application-f59d2.br.js:1:27448
+m@https://discourse-cdn.com/assets/application-f59d2.br.js:1:27560
+string@https://discourse-cdn.com/assets/application-f59d2.br.js:1:27869`
+    );
+    await render(hbs`{{back-trace backtrace=backtrace}}`);
+    const lines = this.backtrace.split("\n");
+    findAll("div.backtrace-line").forEach((node, index) => {
+      assert.equal(node.textContent.trim(), lines[index]);
+    });
+  });
+
+  test("Github links use commit sha", async function(assert) {
+    const backtrace = `/var/www/discourse/lib/permalink_constraint.rb:6:in \`matches?'`;
+    let env = [
+      { application_version: "123abc" },
+      { application_version: "abc123" }
+    ];
+    this.setProperties({
+      backtrace,
+      env
+    });
+    await render(hbs`{{back-trace backtrace=backtrace env=env}}`);
+    let href = find("a").href;
+    assert.equal(
+      href,
+      "https://github.com/discourse/discourse/blob/123abc/lib/permalink_constraint.rb#L6",
+      "uses the first application_version if there are multiple versions"
+    );
+
+    env = { application_version: "567def" };
+    this.set("env", env);
+    await render(hbs`{{back-trace backtrace=backtrace env=env}}`);
+    href = find("a").href;
+    assert.equal(
+      href,
+      "https://github.com/discourse/discourse/blob/567def/lib/permalink_constraint.rb#L6",
+      "uses application_version when env is only a hash"
+    );
+
+    this.set("env", null);
+    await render(hbs`{{back-trace backtrace=backtrace env=env}}`);
+    href = find("a").href;
+    assert.equal(
+      href,
+      "https://github.com/discourse/discourse/blob/ce512452b512b909c38e9c63f2a0e1f8c17a2399/lib/permalink_constraint.rb#L6",
+      "falls back to preload if env doesn't contain application_version"
+    );
+
+    mutatePreload("application_version", null);
+    await render(hbs`{{back-trace backtrace=backtrace}}`);
+    href = find("a").href;
+    assert.equal(
+      href,
+      "https://github.com/discourse/discourse/blob/master/lib/permalink_constraint.rb#L6",
+      "falls back to master branch when neither preload nor application_version in env are available"
+    );
+  });
+});

data/client-app/tests/integration/components/message-info-test.js

@@ -32,11 +32,12 @@ module("Integration | Component | message-info", function(hooks) {
             showTitle=showTitle
             currentEnvPosition=envPosition
             envChangedAction=callback
+            showShare=true
             actionsInMenu=actionsInMenu}}`
     );
     let activeTab = find(".message-info .content.active pre");
     assert.equal(
-      activeTab.textContent,
+      activeTab.textContent.trim(),
       backtrace,
       "default active tab is backtrace"
     );
@@ -60,7 +61,7 @@ module("Integration | Component | message-info", function(hooks) {
     await click(find(".message-actions button.expand.no-text"));
     assert.equal(
       findAll(".actions-menu button").length,
-      2,
+      3,
       "extra buttons shown inside a menu"
     );
     assert
@@ -85,7 +86,7 @@ module("Integration | Component | message-info", function(hooks) {
       .doesNotExist("menu expand button is not shown");
     assert.equal(
       findAll(".message-actions button").length,
-      3,
+      4,
       "all actions buttons are shown inline when `actionsInMenu` is false"
     );
 

data/client-app/tests/integration/components/patterns-list-test.js

@@ -12,7 +12,9 @@ module("Integration | Component | patterns-list", function(hooks) {
       mutable: true,
       patterns: []
     });
-    await render(hbs`{{patterns-list patterns=patterns mutable=mutable}}`);
+    await render(
+      hbs`{{patterns-list patterns=patterns mutable=mutable showCounter=true}}`
+    );
     assert
       .dom(".pattern-input")
       .exists("It shows an input when patterns are emtpy");
@@ -33,7 +35,10 @@ module("Integration | Component | patterns-list", function(hooks) {
       .doesNotExist("No save buttons are shown when there is 0 buffer");
     const counters = findAll("input.count");
     assert.equal(counters.length, 3, "counters shown for all patterns");
-    assert.ok(counters.every(c => c.disabled), "counters are disabled");
+    assert.ok(
+      counters.every(c => c.disabled),
+      "counters are disabled"
+    );
 
     pattern1.set("count", 6);
     this.set("patterns", [pattern1, pattern2]);

data/lib/logster.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require 'logster/version'
 require 'logster/logger'
 require 'logster/message'
 require 'logster/configuration'

data/lib/logster/base_store.rb

@@ -18,7 +18,7 @@ module Logster
     end
 
     # Modify the saved message to the given one (identified by message.key) and bump it to the top of the latest list
-    def replace_and_bump(message, save_env: true)
+    def replace_and_bump(message)
       not_implemented
     end
 
@@ -195,26 +195,33 @@ module Logster
       similar = nil
 
       if Logster.config.allow_grouping
+        message.apply_message_size_limit(
+          Logster.config.maximum_message_size_bytes,
+          gems_dir: Logster.config.gems_dir
+        )
         key = self.similar_key(message)
         similar = get(key, load_env: false) if key
       end
 
+      message.drop_redundant_envs(Logster.config.max_env_count_per_message)
+      message.apply_env_size_limit(Logster.config.max_env_bytes)
+      saved = true
       if similar
-        if similar.count < Logster::MAX_GROUPING_LENGTH
-          similar.env = get_env(similar.key) || {}
-        end
-        save_env = similar.merge_similar_message(message)
-
-        replace_and_bump(similar, save_env: save_env)
+        similar.merge_similar_message(message)
+        replace_and_bump(similar)
         similar
       else
-        save message
+        message.apply_message_size_limit(
+          Logster.config.maximum_message_size_bytes,
+          gems_dir: Logster.config.gems_dir
+        )
+        saved = save(message)
         message
       end
 
       message = similar || message
 
-      if Logster.config.enable_custom_patterns_via_ui || allow_custom_patterns
+      if (Logster.config.enable_custom_patterns_via_ui || allow_custom_patterns) && saved
         grouping_patterns = @patterns_cache.fetch(Logster::GroupingPattern::CACHE_KEY) do
           Logster::GroupingPattern.find_all(store: self)
         end

data/lib/logster/configuration.rb

@@ -12,7 +12,12 @@ module Logster
       :environments,
       :rate_limit_error_reporting,
       :web_title,
-      :maximum_message_size_bytes
+      :maximum_message_size_bytes,
+      :project_directories,
+      :enable_backtrace_links,
+      :gems_dir,
+      :max_env_bytes,
+      :max_env_count_per_message
     )
 
     attr_writer :subdirectory
@@ -26,7 +31,12 @@ module Logster
       @enable_custom_patterns_via_ui = false
       @rate_limit_error_reporting = true
       @enable_js_error_reporting = true
-      @maximum_message_size_bytes = 60_000
+      @maximum_message_size_bytes = 10_000
+      @max_env_bytes = 1000
+      @max_env_count_per_message = 50
+      @project_directories = []
+      @enable_backtrace_links = true
+      @gems_dir = Gem.dir + "/gems/"
 
       @allow_grouping = false
 

data/lib/logster/defer_logger.rb

@@ -7,7 +7,7 @@ module Logster
     private
 
     def report_to_store(severity, progname, message, opts = {})
-      opts[:backtrace] ||= caller
+      opts[:backtrace] ||= caller.join("\n")
       Logster::Scheduler.schedule do
         super(severity, progname, message, opts)
       end

data/lib/logster/logger.rb

@@ -70,6 +70,18 @@ module Logster
         message = message.scrub
       end
 
+      # we want to get the backtrace as early as possible so that logster's
+      # own methods don't show up as the first few frames in the backtrace
+      if !opts || !opts.key?(:backtrace)
+        opts ||= {}
+        backtrace = caller_locations
+        while backtrace.first.path.end_with?("/logger.rb")
+          backtrace.shift
+        end
+        backtrace = backtrace.join("\n")
+        opts[:backtrace] = backtrace
+      end
+
       if @chained
         i = 0
         # micro optimise for logging

data/lib/logster/message.rb

@@ -4,8 +4,6 @@ require 'digest/sha1'
 require 'securerandom'
 
 module Logster
-
-  MAX_GROUPING_LENGTH = 50
   MAX_MESSAGE_LENGTH = 600
 
   class Message
@@ -22,9 +20,10 @@ module Logster
       hostname
       process_id
       application_version
+      time
     }
 
-    attr_accessor :timestamp, :severity, :progname, :key, :backtrace, :count, :protected, :first_timestamp
+    attr_accessor :timestamp, :severity, :progname, :key, :backtrace, :count, :protected, :first_timestamp, :env_buffer
     attr_reader :message, :env
 
     def initialize(severity, progname, message, timestamp = nil, key = nil, count: 1)
@@ -37,6 +36,7 @@ module Logster
       @count = count || 1
       @protected = false
       @first_timestamp = nil
+      @env_buffer = []
     end
 
     def to_h(exclude_env: false)
@@ -82,7 +82,6 @@ module Logster
     end
 
     def env=(env)
-      @env_json = nil
       @env = self.class.scrub_params(env)
     end
 
@@ -94,12 +93,18 @@ module Logster
       env ||= {}
       if Array === env
         env = env.map do |single_env|
-          self.class.default_env.merge(single_env)
+          single_env = self.class.default_env.merge(single_env)
+          if !single_env.key?("time") && !single_env.key?(:time)
+            single_env["time"] = @timestamp || get_timestamp
+          end
+          single_env
         end
       else
         env = self.class.default_env.merge(env)
+        if !env.key?("time") && !env.key?(:time)
+          env["time"] = @timestamp || get_timestamp
+        end
       end
-      @env_json = nil
       @env = Message.populate_from_env(env)
     end
 
@@ -127,12 +132,11 @@ module Logster
       if Array === env
         versions = env.map { |single_env| single_env["application_version"] }
       else
-        versions = env["application_version"]
+        versions = [env["application_version"]]
       end
+      versions.compact!
 
-      if versions && backtrace && backtrace.length > 0
-        versions = [versions] if String === versions
-
+      if backtrace && backtrace.length > 0
         versions.map do |version|
           Digest::SHA1.hexdigest "#{version} #{backtrace}"
         end
@@ -146,31 +150,24 @@ module Logster
     def merge_similar_message(other)
       self.first_timestamp ||= self.timestamp
       self.timestamp = [self.timestamp, other.timestamp].max
-
       self.count += other.count || 1
-      return false if self.count > Logster::MAX_GROUPING_LENGTH
 
-      size = self.to_json(exclude_env: true).bytesize + self.env_json.bytesize
-      extra_env_size = other.env_json.bytesize
-      return false if size + extra_env_size > Logster.config.maximum_message_size_bytes
-
-      other_env = JSON.load JSON.fast_generate other.env
-      if Hash === other_env && !other_env.key?("time")
-        other_env["time"] = other.timestamp
-      end
-      if Hash === self.env && !self.env.key?("time")
-        self.env["time"] = self.first_timestamp
+      if Hash === other.env && !other.env.key?("time") && !other.env.key?(:time)
+        other.env["time"] = other.timestamp
       end
 
-      if Array === self.env
-        Array === other_env ? self.env.concat(other_env) : self.env << other_env
+      if Array === other.env
+        env_buffer.unshift(*other.env)
       else
-        Array === other_env ? self.env = [self.env, *other_env] : self.env = [self.env, other_env]
+        env_buffer.unshift(other.env)
       end
-      @env_json = nil
       true
     end
 
+    def has_env_buffer?
+      env_buffer.size > 0
+    end
+
     def self.populate_from_env(env)
       if Array === env
         env.map do |single_env|
@@ -229,10 +226,6 @@ module Logster
       end
     end
 
-    def env_json
-      @env_json ||= (self.env || {}).to_json
-    end
-
     def self.scrub_params(params)
       if Array === params
         params.map! { |p| scrub_params(p) }
@@ -250,8 +243,74 @@ module Logster
       end
     end
 
+    def drop_redundant_envs(limit)
+      if Array === env
+        env.slice!(limit..-1)
+      end
+    end
+
+    def apply_env_size_limit(size_limit)
+      if Array === env
+        env.each { |e| truncate_env(e, size_limit) }
+      elsif Hash === env
+        truncate_env(env, size_limit)
+      end
+    end
+
+    def apply_message_size_limit(limit, gems_dir: nil)
+      size = self.to_json(exclude_env: true).bytesize
+      if size > limit && @backtrace
+        @backtrace.gsub!(gems_dir, "") if gems_dir
+        @backtrace.strip!
+        size = self.to_json(exclude_env: true).bytesize
+        backtrace_limit = limit - (size - @backtrace.bytesize)
+        return if backtrace_limit <= 0 || size <= limit
+        truncate_backtrace(backtrace_limit)
+      end
+    end
+
+    def truncate_backtrace(bytes_limit)
+      @backtrace = @backtrace.byteslice(0...bytes_limit)
+      while !@backtrace[-1].valid_encoding? && @backtrace.size > 1
+        @backtrace.slice!(-1)
+      end
+    end
+
     protected
 
+    def truncate_env(env, limit)
+      if JSON.fast_generate(env).bytesize > limit
+        sizes = {}
+        braces = '{}'.bytesize
+        env.each do |k, v|
+          sizes[k] = JSON.fast_generate(k => v).bytesize - braces
+        end
+        sorted = env.keys.sort { |a, b| sizes[a] <=> sizes[b] }
+
+        kept_keys = []
+        if env.key?(:time)
+          kept_keys << :time
+        elsif env.key?("time")
+          kept_keys << "time"
+        end
+
+        sum = braces
+        if time_key = kept_keys.first
+          sum += sizes[time_key]
+          sorted.delete(time_key)
+        end
+        comma = ','.bytesize
+
+        sorted.each do |k|
+          extra = kept_keys.size == 0 ? 0 : comma
+          break if sum + sizes[k] + extra > limit
+          kept_keys << k
+          sum += sizes[k] + extra
+        end
+        env.select! { |k| kept_keys.include?(k) }
+      end
+    end
+
     def truncate_message(msg)
       return msg unless msg
       msg = msg.inspect unless String === msg